From 4077c3d84fc964c943166b44e83be3589e80cb59 Mon Sep 17 00:00:00 2001 From: Andrei Ciobanu Date: Mon, 4 Aug 2025 16:39:12 +0300 Subject: [PATCH] Feature branch: Ephemeral resources (#2852) Signed-off-by: Andrei Ciobanu --- internal/addrs/module_instance.go | 11 + internal/addrs/move_endpoint.go | 15 + internal/addrs/move_endpoint_test.go | 52 ++ internal/addrs/parse_ref.go | 20 +- internal/addrs/parse_ref_test.go | 98 ++++ internal/addrs/parse_target.go | 20 +- internal/addrs/parse_target_test.go | 151 +++++ internal/addrs/remove_endpoint.go | 10 + internal/addrs/remove_endpoint_test.go | 46 ++ internal/addrs/resource.go | 38 +- internal/addrs/resource_test.go | 197 +++++++ internal/addrs/resourcemode_string.go | 12 +- internal/builtin/providers/tf/provider.go | 21 + .../provisioners/file/resource_provisioner.go | 1 + .../local-exec/resource_provisioner.go | 1 + .../remote-exec/resource_provisioner.go | 1 + internal/checks/state_init.go | 4 + internal/checks/state_test.go | 41 +- .../happypath/child/checks-happypath-child.tf | 30 +- internal/command/e2etest/primary_test.go | 290 ++++++++++ .../testdata/ephemeral-workflow/main.tf | 89 +++ .../testdata/ephemeral-workflow/mod/main.tf | 15 + internal/command/import.go | 12 +- internal/command/import_test.go | 41 +- internal/command/jsonchecks/checks_test.go | 44 ++ internal/command/jsonchecks/objects.go | 2 + internal/command/jsonconfig/config.go | 7 + internal/command/jsonconfig/config_test.go | 113 +++- internal/command/jsonentities/change.go | 4 + internal/command/jsonformat/plan.go | 12 + internal/command/jsonformat/plan_test.go | 139 ++++- internal/command/jsonformat/state.go | 5 + internal/command/jsonplan/plan.go | 51 +- internal/command/jsonplan/values.go | 2 + internal/command/jsonprovider/provider.go | 18 +- .../command/jsonprovider/provider_test.go | 72 ++- internal/command/jsonstate/state.go | 39 +- internal/command/jsonstate/state_test.go | 152 ++++- internal/command/state_mv.go | 14 +- internal/command/state_mv_test.go | 102 ++++ internal/command/testing/test_provider.go | 1 + internal/command/views/hook_count.go | 5 +- internal/command/views/hook_count_test.go | 32 ++ internal/command/views/hook_json.go | 30 + internal/command/views/hook_json_test.go | 184 +++++++ internal/command/views/hook_ui.go | 116 ++++ internal/command/views/hook_ui_test.go | 98 ++++ internal/command/views/json/hook.go | 52 ++ internal/command/views/json/message_types.go | 22 +- internal/command/views/operation.go | 4 + internal/command/views/operation_test.go | 37 ++ internal/command/views/plan_test.go | 49 ++ internal/communicator/shared/shared.go | 1 + internal/configs/config.go | 72 +-- internal/configs/config_test.go | 139 +++++ internal/configs/configschema/implied_type.go | 18 + internal/configs/configschema/marks.go | 40 ++ internal/configs/configschema/path_test.go | 1 - internal/configs/configschema/schema.go | 7 + internal/configs/module.go | 58 +- internal/configs/module_merge.go | 8 + internal/configs/module_merge_test.go | 122 +++++ internal/configs/module_test.go | 88 ++- internal/configs/moved.go | 17 + internal/configs/moved_test.go | 26 + internal/configs/named_values.go | 33 ++ internal/configs/parser_config.go | 11 + internal/configs/provider_validation.go | 2 + internal/configs/resource.go | 170 ++++++ internal/configs/test_file.go | 1 + .../configs/testdata/ephemeral-blocks/main.tf | 40 ++ .../valid-files/providers-explicit-implied.tf | 4 + .../implied-providers/resources.tf | 11 +- .../nested-providers-fqns/child/main.tf | 9 + .../nested-providers-fqns/main.tf | 9 + .../override-output/a_override.tf | 10 + .../override-output/b_override.tf | 11 + .../valid-modules/override-output/primary.tf | 9 + .../override-variable/a_override.tf | 1 + .../override-variable/b_override.tf | 1 + internal/grpcwrap/provider.go | 101 +++- internal/grpcwrap/provider6.go | 101 +++- internal/lang/eval.go | 130 ++++- internal/lang/eval_test.go | 215 ++++++++ internal/lang/marks/marks.go | 21 + internal/legacy/tofu/resource.go | 3 + internal/legacy/tofu/resource_address.go | 14 + internal/legacy/tofu/schemas.go | 3 + internal/legacy/tofu/state.go | 3 + internal/plans/action.go | 4 + internal/plans/action_string.go | 12 +- internal/plans/changes.go | 16 + internal/plans/changes_test.go | 38 ++ .../plans/internal/planproto/planfile.pb.go | 89 +-- .../plans/internal/planproto/planfile.proto | 1 + internal/plans/objchange/objchange.go | 10 + internal/plans/objchange/plan_valid.go | 3 + internal/plans/planfile/tfplan.go | 11 + internal/plans/planfile/tfplan_test.go | 35 ++ internal/plugin/convert/schema.go | 11 + internal/plugin/grpc_provider.go | 148 ++++- internal/plugin/grpc_provider_test.go | 174 ++++++ internal/plugin6/convert/schema.go | 12 + internal/plugin6/grpc_provider.go | 149 ++++- internal/plugin6/grpc_provider_test.go | 174 ++++++ internal/provider-simple-v6/provider.go | 83 ++- internal/provider-simple/provider.go | 83 ++- internal/providers/deferral.go | 21 +- internal/providers/provider.go | 105 ++++ internal/providers/schemas.go | 3 + internal/repl/session_test.go | 4 +- internal/states/statefile/version3_upgrade.go | 6 + internal/states/statefile/version4.go | 7 + internal/tofu/context_apply2_test.go | 89 ++- internal/tofu/context_input.go | 45 +- internal/tofu/context_input_test.go | 159 +++--- internal/tofu/context_plan2_test.go | 174 +++++- internal/tofu/context_plugins.go | 11 + internal/tofu/context_test.go | 15 + internal/tofu/eval_variable.go | 30 +- internal/tofu/eval_variable_test.go | 64 ++- internal/tofu/evaluate.go | 58 +- internal/tofu/evaluate_test.go | 185 +++++++ internal/tofu/evaluate_valid.go | 53 +- internal/tofu/evaluate_valid_test.go | 24 +- internal/tofu/graph_builder_apply.go | 3 + internal/tofu/graph_builder_plan.go | 27 +- internal/tofu/graph_builder_plan_test.go | 65 +++ internal/tofu/hook.go | 44 ++ internal/tofu/hook_mock.go | 106 ++++ internal/tofu/hook_stop.go | 28 + internal/tofu/hook_test.go | 49 ++ internal/tofu/marks.go | 14 + internal/tofu/node_output.go | 38 ++ internal/tofu/node_output_test.go | 99 ++++ internal/tofu/node_resource_abstract.go | 30 +- .../tofu/node_resource_abstract_instance.go | 517 +++++++++++++++++- internal/tofu/node_resource_apply_instance.go | 40 ++ internal/tofu/node_resource_closeable.go | 58 ++ internal/tofu/node_resource_deposed.go | 2 + internal/tofu/node_resource_destroy.go | 29 +- internal/tofu/node_resource_plan.go | 39 ++ internal/tofu/node_resource_plan_destroy.go | 21 + internal/tofu/node_resource_plan_instance.go | 57 ++ internal/tofu/node_resource_plan_orphan.go | 25 +- .../tofu/node_resource_plan_orphan_test.go | 24 + internal/tofu/node_resource_validate.go | 108 +++- internal/tofu/node_resource_validate_test.go | 199 +++++++ internal/tofu/node_variable_reference.go | 1 + internal/tofu/provider_for_test_framework.go | 19 + internal/tofu/provider_mock.go | 131 ++++- internal/tofu/resource_provider_mock_test.go | 26 +- internal/tofu/schemas_test.go | 4 + internal/tofu/testdata/input-provider/main.tf | 2 + .../static-validate-refs.tf | 3 + .../transform-config-mode-data/main.tf | 2 + .../tofu/transform_attach_config_resource.go | 2 + internal/tofu/transform_closeable_resource.go | 129 +++++ internal/tofu/transform_config.go | 29 +- internal/tofu/transform_config_test.go | 69 ++- internal/tofu/transform_destroy_edge.go | 23 +- internal/tofu/transform_destroy_edge_test.go | 30 +- internal/tofu/transform_diff_test.go | 72 +++ internal/tofu/transform_provider.go | 4 + internal/tofu/transform_reference.go | 104 ++-- internal/tofu/transform_reference_test.go | 99 ++++ 166 files changed, 8044 insertions(+), 565 deletions(-) create mode 100644 internal/command/e2etest/testdata/ephemeral-workflow/main.tf create mode 100644 internal/command/e2etest/testdata/ephemeral-workflow/mod/main.tf create mode 100644 internal/configs/testdata/ephemeral-blocks/main.tf create mode 100644 internal/configs/testdata/valid-modules/override-output/a_override.tf create mode 100644 internal/configs/testdata/valid-modules/override-output/b_override.tf create mode 100644 internal/configs/testdata/valid-modules/override-output/primary.tf create mode 100644 internal/tofu/node_resource_closeable.go create mode 100644 internal/tofu/transform_closeable_resource.go diff --git a/internal/addrs/module_instance.go b/internal/addrs/module_instance.go index 3c573de69a..5784eacc14 100644 --- a/internal/addrs/module_instance.go +++ b/internal/addrs/module_instance.go @@ -82,6 +82,17 @@ func ParseModuleInstanceStr(str string) (ModuleInstance, tfdiags.Diagnostics) { return addr, diags } +// MustParseModuleInstanceStr is a wrapper around ParseModuleInstanceStr that panics if +// it returns an error. +// This is mainly meant for being used in unit tests. +func MustParseModuleInstanceStr(str string) ModuleInstance { + result, diags := ParseModuleInstanceStr(str) + if diags.HasErrors() { + panic(diags.Err().Error()) + } + return result +} + // parseModuleInstancePrefix parses a module instance address from the given // traversal, returning the module instance address and the remaining // traversal. diff --git a/internal/addrs/move_endpoint.go b/internal/addrs/move_endpoint.go index 76d7e761d9..489af2379b 100644 --- a/internal/addrs/move_endpoint.go +++ b/internal/addrs/move_endpoint.go @@ -299,3 +299,18 @@ func (e *MoveEndpoint) internalAddrType() TargetableAddrType { panic(fmt.Sprintf("unsupported address type %T", addr)) } } + +// SubjectAllowed is validating what types of resource can be used with the moved block. +// At the time of writing, it was only ensuring that the moved blocks cannot be used against ephemeral resources. +// This can later be expanded with more rules +func (e *MoveEndpoint) SubjectAllowed() bool { + if e == nil { + return false + } + switch addr := e.relSubject.(type) { + case AbsMoveableResource: + return addr.AffectedAbsResource().Resource.Mode != EphemeralResourceMode + default: + return true + } +} diff --git a/internal/addrs/move_endpoint_test.go b/internal/addrs/move_endpoint_test.go index 29fa7ce56e..35c4251106 100644 --- a/internal/addrs/move_endpoint_test.go +++ b/internal/addrs/move_endpoint_test.go @@ -635,3 +635,55 @@ func TestMoveEndpointConfigMoveable(t *testing.T) { }) } } + +func TestSubjectAllowed(t *testing.T) { + tests := map[string]struct { + target AbsMoveable + want bool + }{ + "ephemeral resource": { + AbsResource{Resource: Resource{Mode: EphemeralResourceMode}}, + false, + }, + "managed resource": { + AbsResource{Resource: Resource{Mode: ManagedResourceMode}}, + true, + }, + "data source": { + AbsResource{Resource: Resource{Mode: DataResourceMode}}, + true, + }, + "ephemeral resource instance": { + AbsResourceInstance{Resource: ResourceInstance{Resource: Resource{Mode: EphemeralResourceMode}}}, + false, + }, + "managed resource instance": { + AbsResourceInstance{Resource: ResourceInstance{Resource: Resource{Mode: ManagedResourceMode}}}, + true, + }, + "data source instance": { + AbsResourceInstance{Resource: ResourceInstance{Resource: Resource{Mode: DataResourceMode}}}, + true, + }, + "module instance": { + ModuleInstance{}, + true, + }, + "module": { + ModuleInstance{}, + true, + }, + "module call": { + AbsModuleCall{}, + true, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + m := &MoveEndpoint{relSubject: tt.target} + if got, want := m.SubjectAllowed(), tt.want; got != want { + t.Errorf("unexpected allowed resource for a moved block. expected: %t; got: %t", want, got) + } + }) + } +} diff --git a/internal/addrs/parse_ref.go b/internal/addrs/parse_ref.go index 00c3c60b5c..63f8917ad5 100644 --- a/internal/addrs/parse_ref.go +++ b/internal/addrs/parse_ref.go @@ -188,6 +188,18 @@ func parseRef(traversal hcl.Traversal) (*Reference, tfdiags.Diagnostics) { } remain := traversal[1:] // trim off "data" so we can use our shared resource reference parser return parseResourceRef(DataResourceMode, rootRange, remain) + case "ephemeral": + if len(traversal) < 3 { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid reference", + Detail: `The "ephemeral" object must be followed by two attribute names: the ephemeral resource type and its name.`, + Subject: traversal.SourceRange().Ptr(), + }) + return nil, diags + } + remain := traversal[1:] // trim off "ephemeral" so we can use our shared resource reference parser + return parseResourceRef(EphemeralResourceMode, rootRange, remain) case "resource": // This is an alias for the normal case of just using a managed resource // type as a top-level symbol, which will serve as an escape mechanism @@ -303,14 +315,16 @@ func parseResourceRef(mode ResourceMode, startRange hcl.Range, traversal hcl.Tra var what string switch mode { case DataResourceMode: - what = "data source" + what = "a data source" + case EphemeralResourceMode: + what = "an ephemeral resource" default: - what = "resource type" + what = "a resource type" } diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid reference", - Detail: fmt.Sprintf(`A reference to a %s must be followed by at least one attribute access, specifying the resource name.`, what), + Detail: fmt.Sprintf(`A reference to %s must be followed by at least one attribute access, specifying the resource name.`, what), Subject: traversal[1].SourceRange().Ptr(), }) return nil, diags diff --git a/internal/addrs/parse_ref_test.go b/internal/addrs/parse_ref_test.go index 82e3915235..7b051ce632 100644 --- a/internal/addrs/parse_ref_test.go +++ b/internal/addrs/parse_ref_test.go @@ -320,6 +320,104 @@ func TestParseRef(t *testing.T) { `The "data" object must be followed by two attribute names: the data source type and the resource name.`, }, + // ephemeral + { + `ephemeral.external.foo`, + &Reference{ + Subject: Resource{ + Mode: EphemeralResourceMode, + Type: "external", + Name: "foo", + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 23, Byte: 22}, + }, + }, + ``, + }, + { + `ephemeral.external.foo.bar`, + &Reference{ + Subject: ResourceInstance{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "external", + Name: "foo", + }, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 23, Byte: 22}, + }, + Remaining: hcl.Traversal{ + hcl.TraverseAttr{ + Name: "bar", + SrcRange: hcl.Range{ + Start: hcl.Pos{Line: 1, Column: 23, Byte: 22}, + End: hcl.Pos{Line: 1, Column: 27, Byte: 26}, + }, + }, + }, + }, + ``, + }, + { + `ephemeral.external.foo["baz"].bar`, + &Reference{ + Subject: ResourceInstance{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "external", + Name: "foo", + }, + Key: StringKey("baz"), + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 30, Byte: 29}, + }, + Remaining: hcl.Traversal{ + hcl.TraverseAttr{ + Name: "bar", + SrcRange: hcl.Range{ + Start: hcl.Pos{Line: 1, Column: 30, Byte: 29}, + End: hcl.Pos{Line: 1, Column: 34, Byte: 33}, + }, + }, + }, + }, + ``, + }, + { + `ephemeral.external.foo["baz"]`, + &Reference{ + Subject: ResourceInstance{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "external", + Name: "foo", + }, + Key: StringKey("baz"), + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 30, Byte: 29}, + }, + }, + ``, + }, + { + `ephemeral`, + nil, + `The "ephemeral" object must be followed by two attribute names: the ephemeral resource type and its name.`, + }, + { + `ephemeral.external`, + nil, + `The "ephemeral" object must be followed by two attribute names: the ephemeral resource type and its name.`, + }, + // local { `local.foo`, diff --git a/internal/addrs/parse_target.go b/internal/addrs/parse_target.go index 64618033b1..6366d38b2e 100644 --- a/internal/addrs/parse_target.go +++ b/internal/addrs/parse_target.go @@ -74,9 +74,13 @@ func parseResourceInstanceUnderModule(moduleAddr ModuleInstance, remain hcl.Trav var diags tfdiags.Diagnostics mode := ManagedResourceMode - if remain.RootName() == "data" { + switch remain.RootName() { + case "data": mode = DataResourceMode remain = remain[1:] + case "ephemeral": + mode = EphemeralResourceMode + remain = remain[1:] } typeName, name, diags := parseResourceTypeAndName(remain, mode) @@ -135,9 +139,14 @@ func parseResourceUnderModule(moduleAddr Module, remain hcl.Traversal) (ConfigRe var diags tfdiags.Diagnostics mode := ManagedResourceMode - if remain.RootName() == "data" { + + switch remain.RootName() { + case "data": mode = DataResourceMode remain = remain[1:] + case "ephemeral": + mode = EphemeralResourceMode + remain = remain[1:] } typeName, name, diags := parseResourceTypeAndName(remain, mode) @@ -205,6 +214,13 @@ func parseResourceTypeAndName(remain hcl.Traversal, mode ResourceMode) (typeName Detail: "A data source name is required.", Subject: remain[0].SourceRange().Ptr(), }) + case EphemeralResourceMode: + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid address", + Detail: "An ephemeral resource name is required.", + Subject: remain[0].SourceRange().Ptr(), + }) default: panic("unknown mode") } diff --git a/internal/addrs/parse_target_test.go b/internal/addrs/parse_target_test.go index a9a007f018..c551f42069 100644 --- a/internal/addrs/parse_target_test.go +++ b/internal/addrs/parse_target_test.go @@ -323,7 +323,158 @@ func TestParseTarget(t *testing.T) { }, ``, }, + // ephemeral + { + `ephemeral.aws_instance.baz`, + &Target{ + Subject: AbsResource{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "aws_instance", + Name: "baz", + }, + Module: RootModuleInstance, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 27, Byte: 26}, + }, + }, + ``, + }, + { + `ephemeral.aws_instance.baz[1]`, + &Target{ + Subject: AbsResourceInstance{ + Resource: ResourceInstance{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "aws_instance", + Name: "baz", + }, + Key: IntKey(1), + }, + Module: RootModuleInstance, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 30, Byte: 29}, + }, + }, + ``, + }, + { + `module.foo.ephemeral.aws_instance.baz`, + &Target{ + Subject: AbsResource{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "aws_instance", + Name: "baz", + }, + Module: ModuleInstance{ + {Name: "foo"}, + }, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 38, Byte: 37}, + }, + }, + ``, + }, + { + `module.foo.module.bar.ephemeral.aws_instance.baz`, + &Target{ + Subject: AbsResource{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "aws_instance", + Name: "baz", + }, + Module: ModuleInstance{ + {Name: "foo"}, + {Name: "bar"}, + }, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 49, Byte: 48}, + }, + }, + ``, + }, + { + `module.foo.module.bar[0].ephemeral.aws_instance.baz`, + &Target{ + Subject: AbsResource{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "aws_instance", + Name: "baz", + }, + Module: ModuleInstance{ + {Name: "foo", InstanceKey: NoKey}, + {Name: "bar", InstanceKey: IntKey(0)}, + }, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 52, Byte: 51}, + }, + }, + ``, + }, + { + `module.foo.module.bar["a"].ephemeral.aws_instance.baz["hello"]`, + &Target{ + Subject: AbsResourceInstance{ + Resource: ResourceInstance{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "aws_instance", + Name: "baz", + }, + Key: StringKey("hello"), + }, + Module: ModuleInstance{ + {Name: "foo", InstanceKey: NoKey}, + {Name: "bar", InstanceKey: StringKey("a")}, + }, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 63, Byte: 62}, + }, + }, + ``, + }, + { + `module.foo.module.bar.ephemeral.aws_instance.baz["hello"]`, + &Target{ + Subject: AbsResourceInstance{ + Resource: ResourceInstance{ + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "aws_instance", + Name: "baz", + }, + Key: StringKey("hello"), + }, + Module: ModuleInstance{ + {Name: "foo"}, + {Name: "bar"}, + }, + }, + SourceRange: tfdiags.SourceRange{ + Start: tfdiags.SourcePos{Line: 1, Column: 1, Byte: 0}, + End: tfdiags.SourcePos{Line: 1, Column: 58, Byte: 57}, + }, + }, + ``, + }, + // errors { `aws_instance`, nil, diff --git a/internal/addrs/remove_endpoint.go b/internal/addrs/remove_endpoint.go index fff457d94d..5a5f671e2b 100644 --- a/internal/addrs/remove_endpoint.go +++ b/internal/addrs/remove_endpoint.go @@ -70,6 +70,16 @@ func ParseRemoveEndpoint(traversal hcl.Traversal) (*RemoveEndpoint, tfdiags.Diag return nil, diags } + if riAddr.Resource.Mode == EphemeralResourceMode { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Ephemeral resource address is not allowed", + Detail: "Ephemeral resources cannot be destroyed, and therefore, 'removed' blocks are not allowed to target them. To remove ephemeral resources from the state, you should remove the ephemeral resource block from the configuration.", + Subject: traversal.SourceRange().Ptr(), + }) + + return nil, diags + } return &RemoveEndpoint{ RelSubject: riAddr, diff --git a/internal/addrs/remove_endpoint_test.go b/internal/addrs/remove_endpoint_test.go index f2f68bd31c..6a95a43649 100644 --- a/internal/addrs/remove_endpoint_test.go +++ b/internal/addrs/remove_endpoint_test.go @@ -177,6 +177,52 @@ func TestParseRemoveEndpoint(t *testing.T) { nil, `Invalid address: A resource name is required.`, }, + // ephemeral + { + `ephemeral.foo.bar`, + nil, + `Ephemeral resource address is not allowed: Ephemeral resources cannot be destroyed, and therefore, 'removed' blocks are not allowed to target them. To remove ephemeral resources from the state, you should remove the ephemeral resource block from the configuration.`, + }, + { + `ephemeral.foo.bar[0]`, + nil, + `Resource instance address with keys is not allowed: Resource address cannot be a resource instance (e.g. "null_resource.a[0]"), it must be a resource instead (e.g. "null_resource.a").`, + }, + { + `ephemeral.foo.bar["a"]`, + nil, + `Resource instance address with keys is not allowed: Resource address cannot be a resource instance (e.g. "null_resource.a[0]"), it must be a resource instead (e.g. "null_resource.a").`, + }, + { + `module.boop.ephemeral.foo.bar[0]`, + nil, + `Resource instance address with keys is not allowed: Resource address cannot be a resource instance (e.g. "null_resource.a[0]"), it must be a resource instead (e.g. "null_resource.a").`, + }, + { + `module.boop.ephemeral.foo.bar["a"]`, + nil, + `Resource instance address with keys is not allowed: Resource address cannot be a resource instance (e.g. "null_resource.a[0]"), it must be a resource instead (e.g. "null_resource.a").`, + }, + { + `module.foo.ephemeral`, + nil, + `Invalid address: Resource specification must include a resource type and name.`, + }, + { + `module.foo.ephemeral.bar`, + nil, + `Invalid address: Resource specification must include a resource type and name.`, + }, + { + `module.foo.ephemeral[0]`, + nil, + `Invalid address: Resource specification must include a resource type and name.`, + }, + { + `module.foo.ephemeral.bar[0]`, + nil, + `Invalid address: A resource name is required.`, + }, } for _, test := range tests { diff --git a/internal/addrs/resource.go b/internal/addrs/resource.go index 000ffab65c..2ca7fe117e 100644 --- a/internal/addrs/resource.go +++ b/internal/addrs/resource.go @@ -29,6 +29,8 @@ func (r Resource) String() string { return fmt.Sprintf("%s.%s", r.Type, r.Name) case DataResourceMode: return fmt.Sprintf("data.%s.%s", r.Type, r.Name) + case EphemeralResourceMode: + return fmt.Sprintf("ephemeral.%s.%s", r.Type, r.Name) default: // Should never happen, but we'll return a string here rather than // crashing just in case it does. @@ -43,7 +45,7 @@ func (r Resource) Equal(o Resource) bool { func (r Resource) Less(o Resource) bool { switch { case r.Mode != o.Mode: - return r.Mode == DataResourceMode + return ResourceModeLess(r.Mode, o.Mode) case r.Type != o.Type: return r.Type < o.Type @@ -511,4 +513,38 @@ const ( // DataResourceMode indicates a data resource, as defined by // "data" blocks in configuration. DataResourceMode ResourceMode = 'D' + + // EphemeralResourceMode indicates an ephemeral resource, as defined by + // the "ephemeral" blocks in configuration. + EphemeralResourceMode ResourceMode = 'E' ) + +// ResourceModeLess is comparing two ResourceMode. +// The ranking is as follows: EphemeralResourceMode, DataResourceMode, ManagedResourceMode. +func ResourceModeLess(a, b ResourceMode) bool { + switch a { + case ManagedResourceMode: + return false // No other mode should be after ManagedResourceMode + case DataResourceMode: + return b == ManagedResourceMode // DataResourceMode is always lower than ManagedResourceMode + case EphemeralResourceMode: + return b == ManagedResourceMode || b == DataResourceMode // EphemeralResourceMode is always lower than ManagedResourceMode and DataResourceMode + } + return false +} + +// ResourceModeBlockName returns the name of the block that the given ResourceMode is mapped to. +// At the time of writing this, the string values returned from this one are hardcoded all over the place so this is not +// the source of truth for the name of those blocks. +func ResourceModeBlockName(rm ResourceMode) string { + switch rm { + case ManagedResourceMode: + return "resource" + case DataResourceMode: + return "data" + case EphemeralResourceMode: + return "ephemeral" + default: + return "unknown" + } +} diff --git a/internal/addrs/resource_test.go b/internal/addrs/resource_test.go index 1267f26318..b4f6c2ef12 100644 --- a/internal/addrs/resource_test.go +++ b/internal/addrs/resource_test.go @@ -7,6 +7,7 @@ package addrs import ( "fmt" + "sort" "testing" "github.com/google/go-cmp/cmp" @@ -26,6 +27,11 @@ func TestResourceEqual_true(t *testing.T) { Type: "a", Name: "b", }, + { + Mode: EphemeralResourceMode, + Type: "a", + Name: "b", + }, } for _, r := range resources { t.Run(r.String(), func(t *testing.T) { @@ -53,6 +59,10 @@ func TestResourceEqual_false(t *testing.T) { Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, Resource{Mode: ManagedResourceMode, Type: "a", Name: "c"}, }, + { + Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Resource{Mode: EphemeralResourceMode, Type: "a", Name: "c"}, + }, } for _, tc := range testCases { t.Run(fmt.Sprintf("%s = %s", tc.left, tc.right), func(t *testing.T) { @@ -85,6 +95,14 @@ func TestResourceInstanceEqual_true(t *testing.T) { }, Key: StringKey("x"), }, + { + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "a", + Name: "b", + }, + Key: StringKey("x"), + }, } for _, r := range resources { t.Run(r.String(), func(t *testing.T) { @@ -110,6 +128,16 @@ func TestResourceInstanceEqual_false(t *testing.T) { Key: IntKey(0), }, }, + { + ResourceInstance{ + Resource: Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Key: IntKey(0), + }, + ResourceInstance{ + Resource: Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + Key: IntKey(0), + }, + }, { ResourceInstance{ Resource: Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, @@ -140,6 +168,26 @@ func TestResourceInstanceEqual_false(t *testing.T) { Key: StringKey("0"), }, }, + { + ResourceInstance{ + Resource: Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Key: IntKey(0), + }, + ResourceInstance{ + Resource: Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Key: StringKey("0"), + }, + }, + { + ResourceInstance{ + Resource: Resource{Mode: DataResourceMode, Type: "a", Name: "b"}, + Key: IntKey(0), + }, + ResourceInstance{ + Resource: Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Key: IntKey(0), + }, + }, } for _, tc := range testCases { t.Run(fmt.Sprintf("%s = %s", tc.left, tc.right), func(t *testing.T) { @@ -157,6 +205,7 @@ func TestResourceInstanceEqual_false(t *testing.T) { func TestAbsResourceInstanceEqual_true(t *testing.T) { managed := Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"} data := Resource{Mode: DataResourceMode, Type: "a", Name: "b"} + ephemeral := Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"} foo, diags := ParseModuleInstanceStr("module.foo") if len(diags) > 0 { @@ -170,7 +219,9 @@ func TestAbsResourceInstanceEqual_true(t *testing.T) { instances := []AbsResourceInstance{ managed.Instance(IntKey(0)).Absolute(foo), data.Instance(IntKey(0)).Absolute(foo), + ephemeral.Instance(IntKey(0)).Absolute(foo), managed.Instance(StringKey("a")).Absolute(foobar), + ephemeral.Instance(IntKey(0)).Absolute(foobar), } for _, r := range instances { t.Run(r.String(), func(t *testing.T) { @@ -184,6 +235,7 @@ func TestAbsResourceInstanceEqual_true(t *testing.T) { func TestAbsResourceInstanceEqual_false(t *testing.T) { managed := Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"} data := Resource{Mode: DataResourceMode, Type: "a", Name: "b"} + ephemeral := Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"} foo, diags := ParseModuleInstanceStr("module.foo") if len(diags) > 0 { @@ -210,6 +262,14 @@ func TestAbsResourceInstanceEqual_false(t *testing.T) { managed.Instance(IntKey(0)).Absolute(foo), managed.Instance(StringKey("0")).Absolute(foo), }, + { + ephemeral.Instance(IntKey(0)).Absolute(foo), + ephemeral.Instance(IntKey(0)).Absolute(foobar), + }, + { + ephemeral.Instance(StringKey("0")).Absolute(foo), + ephemeral.Instance(IntKey(0)).Absolute(foo), + }, } for _, tc := range testCases { t.Run(fmt.Sprintf("%s = %s", tc.left, tc.right), func(t *testing.T) { @@ -305,6 +365,10 @@ func TestConfigResourceEqual_true(t *testing.T) { Resource: Resource{Mode: DataResourceMode, Type: "a", Name: "b"}, Module: RootModule, }, + { + Resource: Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Module: RootModule, + }, { Resource: Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, Module: Module{"foo"}, @@ -313,6 +377,10 @@ func TestConfigResourceEqual_true(t *testing.T) { Resource: Resource{Mode: DataResourceMode, Type: "a", Name: "b"}, Module: Module{"foo"}, }, + { + Resource: Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Module: Module{"foo"}, + }, } for _, r := range resources { t.Run(r.String(), func(t *testing.T) { @@ -326,6 +394,7 @@ func TestConfigResourceEqual_true(t *testing.T) { func TestConfigResourceEqual_false(t *testing.T) { managed := Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"} data := Resource{Mode: DataResourceMode, Type: "a", Name: "b"} + ephemeral := Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"} foo := Module{"foo"} foobar := Module{"foobar"} @@ -337,10 +406,22 @@ func TestConfigResourceEqual_false(t *testing.T) { ConfigResource{Resource: managed, Module: foo}, ConfigResource{Resource: data, Module: foo}, }, + { + ConfigResource{Resource: managed, Module: foo}, + ConfigResource{Resource: ephemeral, Module: foo}, + }, + { + ConfigResource{Resource: data, Module: foo}, + ConfigResource{Resource: ephemeral, Module: foo}, + }, { ConfigResource{Resource: managed, Module: foo}, ConfigResource{Resource: managed, Module: foobar}, }, + { + ConfigResource{Resource: ephemeral, Module: foo}, + ConfigResource{Resource: ephemeral, Module: foobar}, + }, } for _, tc := range testCases { t.Run(fmt.Sprintf("%s = %s", tc.left, tc.right), func(t *testing.T) { @@ -385,6 +466,17 @@ func TestParseConfigResource(t *testing.T) { }, }, }, + { + Input: "ephemeral.a.b", + WantConfigResource: ConfigResource{ + Module: RootModule, + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "a", + Name: "b", + }, + }, + }, { Input: "module.a.b.c", WantConfigResource: ConfigResource{ @@ -407,6 +499,17 @@ func TestParseConfigResource(t *testing.T) { }, }, }, + { + Input: "module.a.ephemeral.b.c", + WantConfigResource: ConfigResource{ + Module: []string{"a"}, + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "b", + Name: "c", + }, + }, + }, { Input: "module.a.module.b.c.d", WantConfigResource: ConfigResource{ @@ -429,6 +532,17 @@ func TestParseConfigResource(t *testing.T) { }, }, }, + { + Input: "module.a.module.b.ephemeral.c.d", + WantConfigResource: ConfigResource{ + Module: []string{"a", "b"}, + Resource: Resource{ + Mode: EphemeralResourceMode, + Type: "c", + Name: "d", + }, + }, + }, { Input: "module.a.module.b", WantErr: "Module address is not allowed: Expected reference to either resource or data block. Provided reference appears to be a module.", @@ -449,6 +563,10 @@ func TestParseConfigResource(t *testing.T) { Input: "module.a.module.b.data.c.d[0]", WantErr: `Resource instance address with keys is not allowed: Resource address cannot be a resource instance (e.g. "null_resource.a[0]"), it must be a resource instead (e.g. "null_resource.a").`, }, + { + Input: "module.a.module.b.ephemeral.c.d[0]", + WantErr: `Resource instance address with keys is not allowed: Resource address cannot be a resource instance (e.g. "null_resource.a[0]"), it must be a resource instead (e.g. "null_resource.a").`, + }, } for _, test := range tests { @@ -485,3 +603,82 @@ func TestParseConfigResource(t *testing.T) { }) } } + +func TestResourceLess(t *testing.T) { + tests := []struct { + left, right Resource + want bool + }{ + { + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + Resource{Mode: DataResourceMode, Type: "a", Name: "b"}, + false, + }, + { + Resource{Mode: DataResourceMode, Type: "a", Name: "b"}, + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + true, + }, + { + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + false, + }, + { + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + Resource{Mode: ManagedResourceMode, Type: "a", Name: "c"}, + true, + }, + { + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + Resource{Mode: ManagedResourceMode, Type: "b", Name: "b"}, + true, + }, + { + Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + true, + }, + { + Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"}, + Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + false, + }, + { + Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + Resource{Mode: DataResourceMode, Type: "a", Name: "b"}, + true, + }, + { + Resource{Mode: DataResourceMode, Type: "a", Name: "b"}, + Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"}, + false, + }, + } + + for _, tt := range tests { + wantComparison := ">" + if tt.want { + wantComparison = "<" + } + t.Run(fmt.Sprintf("%s %s %s", tt.left, wantComparison, tt.right), func(t *testing.T) { + if got, want := tt.left.Less(tt.right), tt.want; got != want { + t.Fatalf("wrong expectation between %q and %q. want: %t; got: %t", tt.left, tt.right, want, got) + } + }) + } +} + +func TestResourceSort(t *testing.T) { + managed := Resource{Mode: ManagedResourceMode, Type: "a", Name: "b"} + data := Resource{Mode: DataResourceMode, Type: "a", Name: "b"} + ephemeral := Resource{Mode: EphemeralResourceMode, Type: "a", Name: "b"} + + got := []Resource{managed, data, ephemeral, managed, ephemeral, data} + sort.SliceStable(got, func(i, j int) bool { return got[i].Less(got[j]) }) + + want := []Resource{ephemeral, ephemeral, data, data, managed, managed} + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("expected no diff meaning that sorting is not working properly.\ndiff: %s", diff) + } +} diff --git a/internal/addrs/resourcemode_string.go b/internal/addrs/resourcemode_string.go index 0b5c33f8ee..a2b727a9b9 100644 --- a/internal/addrs/resourcemode_string.go +++ b/internal/addrs/resourcemode_string.go @@ -11,20 +11,26 @@ func _() { _ = x[InvalidResourceMode-0] _ = x[ManagedResourceMode-77] _ = x[DataResourceMode-68] + _ = x[EphemeralResourceMode-69] } const ( _ResourceMode_name_0 = "InvalidResourceMode" - _ResourceMode_name_1 = "DataResourceMode" + _ResourceMode_name_1 = "DataResourceModeEphemeralResourceMode" _ResourceMode_name_2 = "ManagedResourceMode" ) +var ( + _ResourceMode_index_1 = [...]uint8{0, 16, 37} +) + func (i ResourceMode) String() string { switch { case i == 0: return _ResourceMode_name_0 - case i == 68: - return _ResourceMode_name_1 + case 68 <= i && i <= 69: + i -= 68 + return _ResourceMode_name_1[_ResourceMode_index_1[i]:_ResourceMode_index_1[i+1]] case i == 77: return _ResourceMode_name_2 default: diff --git a/internal/builtin/providers/tf/provider.go b/internal/builtin/providers/tf/provider.go index 7643bb4a70..2373156365 100644 --- a/internal/builtin/providers/tf/provider.go +++ b/internal/builtin/providers/tf/provider.go @@ -78,6 +78,11 @@ func (p *Provider) ValidateDataResourceConfig(_ context.Context, req providers.V return res } +// ValidateEphemeralConfig is used to validate the ephemeral resource configuration values. +func (p *Provider) ValidateEphemeralConfig(context.Context, providers.ValidateEphemeralConfigRequest) providers.ValidateEphemeralConfigResponse { + panic("Should not be called directly, special case for terraform_remote_state") +} + // Configure configures and initializes the provider. func (p *Provider) ConfigureProvider(context.Context, providers.ConfigureProviderRequest) providers.ConfigureProviderResponse { // At this moment there is nothing to configure for the terraform provider, @@ -125,6 +130,22 @@ func (p *Provider) ReadDataSourceEncrypted(ctx context.Context, req providers.Re return res } +// OpenEphemeralResource opens an ephemeral resource returning the ephemeral value returned from the provider. +func (p *Provider) OpenEphemeralResource(context.Context, providers.OpenEphemeralResourceRequest) providers.OpenEphemeralResourceResponse { + panic("Should not be called directly, special case for terraform_remote_state") +} + +// RenewEphemeralResource is renewing an ephemeral resource returning only the private information from the provider. +func (p *Provider) RenewEphemeralResource(context.Context, providers.RenewEphemeralResourceRequest) providers.RenewEphemeralResourceResponse { + panic("Should not be called directly, special case for terraform_remote_state") +} + +// CloseEphemeralResource is closing an ephemeral resource to allow the provider to clean up any possible remote information +// bound to the previously opened ephemeral resource. +func (p *Provider) CloseEphemeralResource(context.Context, providers.CloseEphemeralResourceRequest) providers.CloseEphemeralResourceResponse { + panic("Should not be called directly, special case for terraform_remote_state") +} + // Stop is called when the provider should halt any in-flight actions. func (p *Provider) Stop(_ context.Context) error { log.Println("[DEBUG] terraform provider cannot Stop") diff --git a/internal/builtin/provisioners/file/resource_provisioner.go b/internal/builtin/provisioners/file/resource_provisioner.go index 0c67b67a91..192f43ffc8 100644 --- a/internal/builtin/provisioners/file/resource_provisioner.go +++ b/internal/builtin/provisioners/file/resource_provisioner.go @@ -53,6 +53,7 @@ func (p *provisioner) GetSchema() (resp provisioners.GetSchemaResponse) { Required: true, }, }, + Ephemeral: true, } resp.Provisioner = schema return resp diff --git a/internal/builtin/provisioners/local-exec/resource_provisioner.go b/internal/builtin/provisioners/local-exec/resource_provisioner.go index 43d282cd7b..c3da02790a 100644 --- a/internal/builtin/provisioners/local-exec/resource_provisioner.go +++ b/internal/builtin/provisioners/local-exec/resource_provisioner.go @@ -67,6 +67,7 @@ func (p *provisioner) GetSchema() (resp provisioners.GetSchemaResponse) { Optional: true, }, }, + Ephemeral: true, } resp.Provisioner = schema diff --git a/internal/builtin/provisioners/remote-exec/resource_provisioner.go b/internal/builtin/provisioners/remote-exec/resource_provisioner.go index 7a8a5bf5a5..e5b3cb1325 100644 --- a/internal/builtin/provisioners/remote-exec/resource_provisioner.go +++ b/internal/builtin/provisioners/remote-exec/resource_provisioner.go @@ -55,6 +55,7 @@ func (p *provisioner) GetSchema() (resp provisioners.GetSchemaResponse) { Optional: true, }, }, + Ephemeral: true, } resp.Provisioner = schema diff --git a/internal/checks/state_init.go b/internal/checks/state_init.go index d4abc8a5cd..dc07ef187f 100644 --- a/internal/checks/state_init.go +++ b/internal/checks/state_init.go @@ -35,6 +35,10 @@ func collectInitialStatuses(into addrs.Map[addrs.ConfigCheckable, *configCheckab addr := rc.Addr().InModule(moduleAddr) collectInitialStatusForResource(into, addr, rc) } + for _, rc := range cfg.Module.EphemeralResources { + addr := rc.Addr().InModule(moduleAddr) + collectInitialStatusForResource(into, addr, rc) + } for _, oc := range cfg.Module.Outputs { addr := oc.Addr().InModule(moduleAddr) diff --git a/internal/checks/state_test.go b/internal/checks/state_test.go index e0745b2ba9..c04c4ea37c 100644 --- a/internal/checks/state_test.go +++ b/internal/checks/state_test.go @@ -65,6 +65,16 @@ func TestChecksHappyPath(t *testing.T) { Type: "null_resource", Name: "c", }.InModule(moduleChild) + dataFoo := addrs.Resource{ + Mode: addrs.DataResourceMode, + Type: "aws_s3_object", + Name: "foo", + }.InModule(moduleChild) + ephemeralBar := addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "aws_secretsmanager_secret_version", + Name: "bar", + }.InModule(moduleChild) childOutput := addrs.OutputValue{ Name: "b", }.InModule(moduleChild) @@ -80,6 +90,15 @@ func TestChecksHappyPath(t *testing.T) { if addr := resourceB; cfg.Children["child"].Module.ResourceByAddr(addr.Resource) == nil { t.Fatalf("configuration does not include %s", addr) } + if addr := resourceC; cfg.Children["child"].Module.ResourceByAddr(addr.Resource) == nil { + t.Fatalf("configuration does not include %s", addr) + } + if addr := dataFoo; cfg.Children["child"].Module.ResourceByAddr(addr.Resource) == nil { + t.Fatalf("configuration does not include %s", addr) + } + if addr := ephemeralBar; cfg.Children["child"].Module.ResourceByAddr(addr.Resource) == nil { + t.Fatalf("configuration does not include %s", addr) + } if addr := resourceNoChecks; cfg.Module.ResourceByAddr(addr.Resource) == nil { t.Fatalf("configuration does not include %s", addr) } @@ -107,6 +126,14 @@ func TestChecksHappyPath(t *testing.T) { t.Errorf("checks not detected for %s", addr) missing++ } + if addr := dataFoo; !checks.ConfigHasChecks(addr) { + t.Errorf("checks not detected for %s", addr) + missing++ + } + if addr := ephemeralBar; !checks.ConfigHasChecks(addr) { + t.Errorf("checks not detected for %s", addr) + missing++ + } if addr := rootOutput; !checks.ConfigHasChecks(addr) { t.Errorf("checks not detected for %s", addr) missing++ @@ -138,6 +165,8 @@ func TestChecksHappyPath(t *testing.T) { resourceA, resourceB, resourceC, + dataFoo, + ephemeralBar, rootOutput, childOutput, checkBlock, @@ -168,6 +197,8 @@ func TestChecksHappyPath(t *testing.T) { moduleChildInst := addrs.RootModuleInstance.Child("child", addrs.NoKey) resourceInstB := resourceB.Resource.Absolute(moduleChildInst).Instance(addrs.NoKey) resourceInstC0 := resourceC.Resource.Absolute(moduleChildInst).Instance(addrs.IntKey(0)) + dataInstFoo := dataFoo.Resource.Absolute(moduleChildInst).Instance(addrs.NoKey) + ephemeralInstBar := ephemeralBar.Resource.Absolute(moduleChildInst).Instance(addrs.NoKey) resourceInstC1 := resourceC.Resource.Absolute(moduleChildInst).Instance(addrs.IntKey(1)) childOutputInst := childOutput.OutputValue.Absolute(moduleChildInst) checkBlockInst := checkBlock.Check.Absolute(addrs.RootModuleInstance) @@ -184,6 +215,12 @@ func TestChecksHappyPath(t *testing.T) { checks.ReportCheckResult(resourceInstC0, addrs.ResourcePostcondition, 0, StatusPass) checks.ReportCheckResult(resourceInstC1, addrs.ResourcePostcondition, 0, StatusPass) + checks.ReportCheckableObjects(dataFoo, addrs.MakeSet[addrs.Checkable](dataInstFoo)) + checks.ReportCheckResult(dataInstFoo, addrs.ResourcePrecondition, 0, StatusPass) + + checks.ReportCheckableObjects(ephemeralBar, addrs.MakeSet[addrs.Checkable](ephemeralInstBar)) + checks.ReportCheckResult(ephemeralInstBar, addrs.ResourcePrecondition, 0, StatusPass) + checks.ReportCheckableObjects(childOutput, addrs.MakeSet[addrs.Checkable](childOutputInst)) checks.ReportCheckResult(childOutputInst, addrs.OutputPrecondition, 0, StatusPass) @@ -206,7 +243,7 @@ func TestChecksHappyPath(t *testing.T) { t.Errorf("incorrect final aggregate check status for %s: %s, but want %s", configAddr, got, want) } } - if got, want := configCount, 6; got != want { + if got, want := configCount, 8; got != want { t.Errorf("incorrect number of known config addresses %d; want %d", got, want) } } @@ -218,6 +255,8 @@ func TestChecksHappyPath(t *testing.T) { resourceInstB, resourceInstC0, resourceInstC1, + dataInstFoo, + ephemeralInstBar, childOutputInst, checkBlockInst, ) diff --git a/internal/checks/testdata/happypath/child/checks-happypath-child.tf b/internal/checks/testdata/happypath/child/checks-happypath-child.tf index d067bc2aa0..5b9bdadde8 100644 --- a/internal/checks/testdata/happypath/child/checks-happypath-child.tf +++ b/internal/checks/testdata/happypath/child/checks-happypath-child.tf @@ -1,3 +1,11 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.81.0" + } + } +} resource "null_resource" "b" { lifecycle { precondition { @@ -18,6 +26,27 @@ resource "null_resource" "c" { } } +data "aws_s3_object" "foo" { + lifecycle { + precondition { + condition = self.id == "" + error_message = "Impossible data." + } + } + bucket = "test-bucket" + key = "test-key" +} + +ephemeral "aws_secretsmanager_secret_version" "bar" { + lifecycle { + precondition { + condition = self.id == "" + error_message = "Impossible ephemeral." + } + } + secret_id = "secret-manager-id" +} + output "b" { value = null_resource.b.id @@ -26,4 +55,3 @@ output "b" { error_message = "B has no id." } } - diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index c4f0c8a3ad..3a90c45495 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -6,14 +6,20 @@ package e2etest import ( + "fmt" + "os" "path/filepath" "reflect" + "runtime" + "slices" "sort" "strings" "testing" "github.com/davecgh/go-spew/spew" + "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/e2e" + "github.com/opentofu/opentofu/internal/getproviders" "github.com/opentofu/opentofu/internal/plans" "github.com/zclconf/go-cty/cty" ) @@ -232,3 +238,287 @@ func TestPrimaryChdirOption(t *testing.T) { t.Errorf("incorrect destroy tally; want 0 destroyed:\n%s", stdout) } } + +// This test is checking the workflow of the ephemeral resources. +// Check also the configuration files for comments. The idea is that at the time of +// writing, the configuration was done in such a way to fail later when the +// marks will be introduced for ephemeral values. Therefore, this test will +// fail later and will require adjustments. +// +// We want to validate that the plan file, state file and the output contain +// only the things that are needed: +// - The plan file needs to contain **only** the stubs of the ephemeral resources +// and not the values that it generated. This is needed for `tofu apply planfile` +// to be able to generate the execution node graphs correctly. +// - The state file must not contain the ephemeral resources changes. +// - The output should contain no changes related to ephemeral resources, but only +// the status update of their execution. +func TestEphemeralWorkflowAndOutput(t *testing.T) { + t.Parallel() + + skipIfCannotAccessNetwork(t) + pluginVersionRunner := func(t *testing.T, testdataPath string, providerBuilderFunc func(*testing.T, string)) { + tf := e2e.NewBinary(t, tofuBin, testdataPath) + providerBuilderFunc(t, tf.WorkDir()) + + { //// INIT + _, stderr, err := tf.Run("init", "-plugin-dir=cache") + if err != nil { + t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) + } + } + + { //// PLAN + stdout, stderr, err := tf.Run("plan", "-out=tfplan") + if err != nil { + t.Fatalf("unexpected plan error: %s\nstderr:\n%s", err, stderr) + } + // TODO ephemeral - this "value_wo" should be shown something like (write-only attribute). This will be handled during the work on the write-only attributes. + // TODO ephemeral - "out_ephemeral" should fail later when the marking of the outputs is implemented fully, so that should not be visible in the output + expectedChangesOutput := `OpenTofu used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + + create + <= read (data resources) + +OpenTofu will perform the following actions: + + # data.simple_resource.test_data2 will be read during apply + # (depends on a resource or a module with changes pending) + <= data "simple_resource" "test_data2" { + + id = (known after apply) + + value = "test" + } + + # simple_resource.test_res will be created + + resource "simple_resource" "test_res" { + + value = "test value" + } + + # simple_resource.test_res_second_provider will be created + + resource "simple_resource" "test_res_second_provider" { + + value = "just a simple resource to ensure that the second provider it's working fine" + } + +Plan: 2 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + final_output = "just a simple resource to ensure that the second provider it's working fine" + + out_ephemeral = "rawvalue"` + + expectedResourcesUpdates := map[string]bool{ + "data.simple_resource.test_data1: Reading...": true, + "data.simple_resource.test_data1: Read complete after": true, + "ephemeral.simple_resource.test_ephemeral[0]: Opening...": true, + "ephemeral.simple_resource.test_ephemeral[0]: Open complete after": true, + "ephemeral.simple_resource.test_ephemeral[1]: Opening...": true, + "ephemeral.simple_resource.test_ephemeral[1]: Open complete after": true, + "ephemeral.simple_resource.test_ephemeral[0]: Closing...": true, + "ephemeral.simple_resource.test_ephemeral[0]: Close complete after": true, + "ephemeral.simple_resource.test_ephemeral[1]: Closing...": true, + "ephemeral.simple_resource.test_ephemeral[1]: Close complete after": true, + } + out := stripAnsi(stdout) + + if !strings.Contains(out, expectedChangesOutput) { + t.Errorf("wrong plan output:\nstdout:%s\nstderr:%s", stdout, stderr) + } + + for reg, required := range expectedResourcesUpdates { + if strings.Contains(out, reg) { + continue + } + if required { + t.Errorf("plan output does not contain required content %q\nout:%s", reg, out) + } else { + // We don't want to fail the test for outputs that are performance and time dependent + // as the renew status updates + t.Logf("plan output does not contain %q\nout:%s", reg, out) + } + } + + // assert plan file content + plan, err := tf.Plan("tfplan") + if err != nil { + t.Fatalf("failed to read the plan file: %s", err) + } + idx := slices.IndexFunc(plan.Changes.Resources, func(src *plans.ResourceInstanceChangeSrc) bool { + return src.Addr.Resource.Resource.Mode == addrs.EphemeralResourceMode + }) + if idx < 0 { + t.Fatalf("no ephemeral resource found in the plan file") + } + res := plan.Changes.Resources[idx] + if res.Before != nil { + t.Errorf("ephemeral resource %q from plan contains before value but it shouldn't: %s", res.Addr.String(), res.Before) + } + if res.After != nil { + t.Errorf("ephemeral resource %q from plan contains after value but it shouldn't: %s", res.Addr.String(), res.After) + } + if got, want := res.Action, plans.Open; got != want { + t.Errorf("ephemeral resource %q from plan contains wrong actions. want %q; got %q", res.Addr.String(), want, got) + } + } + + { //// APPLY + stdout, stderr, err := tf.Run("apply", "tfplan") + if err != nil { + t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) + } + state, err := tf.LocalState() + if err != nil { + t.Fatalf("failed to read local state: %s", err) + } + expectedResources := map[string]bool{ + "data.simple_resource.test_data1": true, + "data.simple_resource.test_data2": true, + "simple_resource.test_res": true, + "simple_resource.test_res_second_provider": true, + "ephemeral.simple_resource.test_ephemeral": false, + } + for res, exists := range expectedResources { + _, ok := state.RootModule().Resources[res] + if ok != exists { + t.Errorf("expected resource %q existence to be %t but got %t", res, exists, ok) + } + } + + expectedChangesOutput := `Apply complete! Resources: 2 added, 0 changed, 0 destroyed.` + // NOTE: the non-required ones are dependent on the performance of the platform that this test is running on. + // In CI, if we would make this as required, this test might be flaky. + expectedResourcesUpdates := map[string]bool{ + "ephemeral.simple_resource.test_ephemeral[0]: Opening...": true, + "ephemeral.simple_resource.test_ephemeral[0]: Open complete after": true, + "ephemeral.simple_resource.test_ephemeral[1]: Opening...": true, + "ephemeral.simple_resource.test_ephemeral[1]: Open complete after": true, + "data.simple_resource.test_data2: Reading...": true, + "data.simple_resource.test_data2: Read complete after": true, + "simple_resource.test_res: Creating...": true, + "simple_resource.test_res_second_provider: Creating...": true, + "simple_resource.test_res_second_provider: Creation complete after": true, + "ephemeral.simple_resource.test_ephemeral[0]: Renewing...": false, + "ephemeral.simple_resource.test_ephemeral[0]: Renew complete after": false, + "ephemeral.simple_resource.test_ephemeral[1]: Renewing...": false, + "ephemeral.simple_resource.test_ephemeral[1]: Renew complete after": false, + "simple_resource.test_res: Creation complete after": true, + "ephemeral.simple_resource.test_ephemeral[0]: Closing...": true, + "ephemeral.simple_resource.test_ephemeral[0]: Close complete after": true, + "ephemeral.simple_resource.test_ephemeral[1]: Closing...": true, + "simple_resource.test_res: Provisioning with 'local-exec'...": true, + `simple_resource.test_res (local-exec): Executing: ["/bin/sh" "-c" "echo \"visible test value\""]`: true, + "simple_resource.test_res (local-exec): visible test value": true, + "simple_resource.test_res (local-exec): (output suppressed due to ephemeral value in config)": true, + } + out := stripAnsi(stdout) + + if !strings.Contains(out, expectedChangesOutput) { + t.Errorf("wrong apply output:\nstdout:%s\nstderr%s", stdout, stderr) + } + + for reg, required := range expectedResourcesUpdates { + if strings.Contains(out, reg) { + continue + } + if required { + t.Errorf("apply output does not contain required content %q\nout:%s", reg, out) + } else { + // We don't want to fail the test for outputs that are performance and time dependent + // as the renew status updates + t.Logf("apply output does not contain %q\nout:%s", reg, out) + } + } + } + { //// DESTROY + stdout, stderr, err := tf.Run("destroy", "-auto-approve") + if err != nil { + t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "Resources: 2 destroyed") { + t.Errorf("incorrect destroy tally; want 2 destroyed:\n%s", stdout) + } + + state, err := tf.LocalState() + if err != nil { + t.Fatalf("failed to read state file after destroy: %s", err) + } + + stateResources := state.RootModule().Resources + if len(stateResources) != 0 { + t.Errorf("wrong resources in state after destroy; want none, but still have:%s", spew.Sdump(stateResources)) + } + } + } + + cases := map[string]struct { + protoBinBuilder func(t *testing.T, workdir string) + }{ + "proto version 5": { + protoBinBuilder: func(t *testing.T, workdir string) { + buildSimpleProvider(t, "5", workdir, "simple") + }, + }, + "proto version 6": { + protoBinBuilder: func(t *testing.T, workdir string) { + buildSimpleProvider(t, "6", workdir, "simple") + }, + }, + } + for name, tt := range cases { + t.Run(name, func(t *testing.T) { + pluginVersionRunner(t, "testdata/ephemeral-workflow", tt.protoBinBuilder) + }) + } +} + +// This function builds and moves to a directory called "cache" inside the workdir, +// the version of the provider passed as argument. +// Instead of using this function directly, the pre-configured functions buildV5TestProvider and +// buildV6TestProvider can be used. +func buildSimpleProvider(t *testing.T, version string, workdir string, buildOutName string) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } + + var ( + providerBinFileName string + implPkgName string + ) + switch version { + case "5": + providerBinFileName = "simple" + implPkgName = "provider-simple" + case "6": + providerBinFileName = "simple6" + implPkgName = "provider-simple-v6" + default: + t.Fatalf("invalid version for simple provider") + } + if buildOutName != "" { + providerBinFileName = buildOutName + } + providerBuildOutDir := filepath.Join(workdir, fmt.Sprintf("terraform-provider-%s", providerBinFileName)) + providerTmpBinPath := e2e.GoBuild(fmt.Sprintf("github.com/opentofu/opentofu/internal/%s/main", implPkgName), providerBuildOutDir) + + extension := "" + if runtime.GOOS == "windows" { + extension = ".exe" + } + + // Move the provider binaries into a directory that we will point tofu + // to using the -plugin-dir cli flag. + platform := getproviders.CurrentPlatform.String() + hashiDir := "cache/registry.opentofu.org/hashicorp/" + providerCacheDir := filepath.Join(workdir, hashiDir, fmt.Sprintf("%s/0.0.1/", providerBinFileName), platform) + if err := os.MkdirAll(providerCacheDir, os.ModePerm); err != nil { + t.Fatal(err) + } + providerFinalBinaryFilePath := filepath.Join(workdir, hashiDir, fmt.Sprintf("%s/0.0.1/", providerBinFileName), platform, fmt.Sprintf("terraform-provider-%s", providerBinFileName)) + extension + if err := os.Rename(providerTmpBinPath, providerFinalBinaryFilePath); err != nil { + t.Fatal(err) + } +} diff --git a/internal/command/e2etest/testdata/ephemeral-workflow/main.tf b/internal/command/e2etest/testdata/ephemeral-workflow/main.tf new file mode 100644 index 0000000000..f1c975b4c8 --- /dev/null +++ b/internal/command/e2etest/testdata/ephemeral-workflow/main.tf @@ -0,0 +1,89 @@ +// the provider-plugin tests uses the -plugin-cache flag so terraform pulls the +// test binaries instead of reaching out to the registry. +terraform { + required_providers { + simple = { + source = "registry.opentofu.org/hashicorp/simple" + } + } +} + +provider "simple" { + alias = "s1" +} + +data "simple_resource" "test_data1" { + provider = simple.s1 + value = "initial data value" +} + +ephemeral "simple_resource" "test_ephemeral" { + count = 2 + provider = simple.s1 + // Having that "-with-renew" suffix, later when this value will be passed into "simple_resource.test_res.value_wo", + // the plugin will delay the response on some requests to allow ephemeral Renew calls to be performed. + value = "${data.simple_resource.test_data1.value}-with-renew" +} + +resource "simple_resource" "test_res" { + provider = simple.s1 + value = "test value" + // NOTE write-only arguments can reference ephemeral values. + value_wo = ephemeral.simple_resource.test_ephemeral[0].value + provisioner "local-exec" { + command = "echo \"visible ${self.value}\"" + } + provisioner "local-exec" { + command = "echo \"not visible ${ephemeral.simple_resource.test_ephemeral[0].value}\"" + } + // NOTE: value_wo cannot be used in a provisioner because it is returned as null by the provider so the interpolation fails +} + +data "simple_resource" "test_data2" { + provider = simple.s1 + value = "test" + lifecycle { + precondition { + // NOTE: precondition blocks can reference ephemeral values + condition = ephemeral.simple_resource.test_ephemeral[0].value != null + error_message = "test message" + } + } +} + +locals{ + simple_provider_cfg = ephemeral.simple_resource.test_ephemeral[0].value +} + +provider "simple" { + alias = "s2" + // NOTE: Ensure that ephemeral values can be used to configure a provider. + // This is needed in two cases: during plan/apply and also during destroy. + // This test has been updated when DestroyEdgeTransformer was updated to + // not create dependencies between ephemeral resources and the destroy nodes. + // The "i_depend_on" field is just a simple configuration attribute of the provider + // to allow creation of dependencies between a resources from a previously + // initialized provider and the provider that is configured here. + // The "i_depend_on" field is having no functionality behind, in the provider context, + // but it's just a way for the "provider" block to create depedencies + // to other blocks. + i_depend_on = local.simple_provider_cfg +} + +resource "simple_resource" "test_res_second_provider" { + provider = simple.s2 + value = "just a simple resource to ensure that the second provider it's working fine" +} + +module "call" { + source = "./mod" + in = ephemeral.simple_resource.test_ephemeral[0].value // NOTE: because variable "in" is marked as ephemeral, this should work as expected. +} + +output "out_ephemeral" { + value = module.call.out2 // TODO: Because the output ephemeral marking is not done yet entirely, this is working now but remove this output once the marking of outputs are done completely. +} + +output "final_output" { + value = simple_resource.test_res_second_provider.value +} \ No newline at end of file diff --git a/internal/command/e2etest/testdata/ephemeral-workflow/mod/main.tf b/internal/command/e2etest/testdata/ephemeral-workflow/mod/main.tf new file mode 100644 index 0000000000..19aa0d49f7 --- /dev/null +++ b/internal/command/e2etest/testdata/ephemeral-workflow/mod/main.tf @@ -0,0 +1,15 @@ +variable "in" { + type = string + description = "Variable that is marked as ephemeral and doesn't matter what value is given in, ephemeral or not, the value evaluated for this variable will be marked as ephemeral" + ephemeral = true +} + +output "out1" { + value = var.in + ephemeral = true // NOTE: because +} + +output "out2" { + value = "rawvalue" // TODO ephemeral - this is returning a raw value and since incomplete work, the evaluated value is not marked as ephemeral. Once this will be fixed, the test should fail + ephemeral = true +} \ No newline at end of file diff --git a/internal/command/import.go b/internal/command/import.go index e640ad223c..f3fd0c48f0 100644 --- a/internal/command/import.go +++ b/internal/command/import.go @@ -7,7 +7,6 @@ package command import ( "context" - "errors" "fmt" "log" "os" @@ -87,7 +86,16 @@ func (c *ImportCommand) Run(args []string) int { } if addr.Resource.Resource.Mode != addrs.ManagedResourceMode { - diags = diags.Append(errors.New("A managed resource address is required. Importing into a data resource is not allowed.")) + var what string + switch addr.Resource.Resource.Mode { + case addrs.DataResourceMode: + what = "a data resource" + case addrs.EphemeralResourceMode: + what = "an ephemeral resource" + default: + what = "a resource type" + } + diags = diags.Append(fmt.Errorf("A managed resource address is required. Importing into %s is not allowed.", what)) c.showDiagnostics(diags) return 1 } diff --git a/internal/command/import_test.go b/internal/command/import_test.go index 38da686549..5ca8c4d014 100644 --- a/internal/command/import_test.go +++ b/internal/command/import_test.go @@ -883,7 +883,7 @@ func TestImportModuleInputVariableEvaluation(t *testing.T) { } } -func TestImport_dataResource(t *testing.T) { +func TestImport_nonManagedResource(t *testing.T) { t.Chdir(testFixturePath("import-missing-resource-config")) statePath := testTempFile(t) @@ -899,19 +899,36 @@ func TestImport_dataResource(t *testing.T) { }, } - args := []string{ - "-state", statePath, - "data.test_data_source.foo", - "bar", - } - code := c.Run(args) - if code != 1 { - t.Fatalf("import succeeded; expected failure") + cases := []struct { + resAddr string + expectedErrMsg string + }{ + { + resAddr: "data.test_data_source.foo", + expectedErrMsg: "A managed resource address is required. Importing into a data resource is not allowed.", + }, + { + resAddr: "ephemeral.test_data_source.foo", + expectedErrMsg: "A managed resource address is required. Importing into an ephemeral resource is not allowed.", + }, } + for _, tt := range cases { + t.Run(tt.resAddr, func(t *testing.T) { + args := []string{ + "-state", statePath, + tt.resAddr, + "bar", + } + code := c.Run(args) + if code != 1 { + t.Fatalf("import succeeded; expected failure") + } - msg := ui.ErrorWriter.String() - if want := `A managed resource address is required`; !strings.Contains(msg, want) { - t.Errorf("incorrect message\nwant substring: %s\ngot:\n%s", want, msg) + msg := ui.ErrorWriter.String() + if want := tt.expectedErrMsg; !strings.Contains(msg, want) { + t.Errorf("incorrect message\nwant substring: %s\ngot:\n%s", want, msg) + } + }) } } diff --git a/internal/command/jsonchecks/checks_test.go b/internal/command/jsonchecks/checks_test.go index 1f23348dbe..4b2f70113c 100644 --- a/internal/command/jsonchecks/checks_test.go +++ b/internal/command/jsonchecks/checks_test.go @@ -43,6 +43,16 @@ func TestMarshalCheckStates(t *testing.T) { outputBInstAddr := addrs.Checkable(addrs.OutputValue{Name: "b"}.Absolute(moduleChildAddr)) checkBlockAAddr := addrs.ConfigCheckable(addrs.Check{Name: "a"}.InModule(addrs.RootModule)) checkBlockAInstAddr := addrs.Checkable(addrs.Check{Name: "a"}.Absolute(addrs.RootModuleInstance)) + ephemeralAAddr := addrs.ConfigCheckable(addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test", + Name: "a", + }.InModule(addrs.RootModule)) + ephemeralAInstAddr := addrs.Checkable(addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test", + Name: "a", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance)) tests := map[string]struct { Input *states.CheckResults @@ -108,6 +118,17 @@ func TestMarshalCheckStates(t *testing.T) { }), ), }), + addrs.MakeMapElem(ephemeralAAddr, &states.CheckResultAggregate{ + Status: checks.StatusFail, + ObjectResults: addrs.MakeMap( + addrs.MakeMapElem(ephemeralAInstAddr, &states.CheckResultObject{ + Status: checks.StatusFail, + FailureMessages: []string{ + "foo", + }, + }), + ), + }), ), }, []any{ @@ -132,6 +153,29 @@ func TestMarshalCheckStates(t *testing.T) { }, "status": "fail", }, + map[string]any{ + "address": map[string]any{ + "kind": "resource", + "mode": "ephemeral", + "name": "a", + "to_display": "ephemeral.test.a", + "type": "test", + }, + "instances": []any{ + map[string]any{ + "address": map[string]any{ + "to_display": `ephemeral.test.a`, + }, + "problems": []any{ + map[string]any{ + "message": "foo", + }, + }, + "status": "fail", + }, + }, + "status": "fail", + }, map[string]any{ "address": map[string]any{ "kind": "output_value", diff --git a/internal/command/jsonchecks/objects.go b/internal/command/jsonchecks/objects.go index fee778ae9a..76bf27e302 100644 --- a/internal/command/jsonchecks/objects.go +++ b/internal/command/jsonchecks/objects.go @@ -31,6 +31,8 @@ func makeStaticObjectAddr(addr addrs.ConfigCheckable) staticObjectAddr { ret["mode"] = "managed" case addrs.DataResourceMode: ret["mode"] = "data" + case addrs.EphemeralResourceMode: + ret["mode"] = "ephemeral" default: panic(fmt.Sprintf("unsupported resource mode %#v", addr.Resource.Mode)) } diff --git a/internal/command/jsonconfig/config.go b/internal/command/jsonconfig/config.go index 32de0e6c9a..96b0e6115c 100644 --- a/internal/command/jsonconfig/config.go +++ b/internal/command/jsonconfig/config.go @@ -341,8 +341,13 @@ func marshalModule(c *configs.Config, schemas *tofu.Schemas, addr string) (modul if err != nil { return module, err } + ephemeralResources, err := marshalResources(c.Module.EphemeralResources, schemas, addr) + if err != nil { + return module, err + } rs = append(managedResources, dataResources...) + rs = append(rs, ephemeralResources...) module.Resources = rs outputs := make(map[string]output) @@ -520,6 +525,8 @@ func marshalResources(resources map[string]*configs.Resource, schemas *tofu.Sche r.Mode = "managed" case addrs.DataResourceMode: r.Mode = "data" + case addrs.EphemeralResourceMode: + r.Mode = "ephemeral" default: return rs, fmt.Errorf("resource %s has an unsupported mode %s", r.Address, v.Mode.String()) } diff --git a/internal/command/jsonconfig/config_test.go b/internal/command/jsonconfig/config_test.go index 29eb2f9aa3..05dcfce5c7 100644 --- a/internal/command/jsonconfig/config_test.go +++ b/internal/command/jsonconfig/config_test.go @@ -10,11 +10,13 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/zclconf/go-cty/cty" - + "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/configs" + "github.com/opentofu/opentofu/internal/configs/configschema" + "github.com/opentofu/opentofu/internal/providers" "github.com/opentofu/opentofu/internal/tofu" + "github.com/zclconf/go-cty/cty" ) func TestFindSourceProviderConfig(t *testing.T) { @@ -114,6 +116,20 @@ func TestFindSourceProviderConfig(t *testing.T) { func TestMarshalModule(t *testing.T) { emptySchemas := &tofu.Schemas{} + providerAddr := addrs.NewProvider("host", "namespace", "type") + resSchema := map[string]providers.Schema{ + "test_type": { + Version: 0, + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "foo": { + Type: cty.String, + Optional: true, + }, + }, + }, + }, + } tests := map[string]struct { Input *configs.Config @@ -249,6 +265,90 @@ func TestMarshalModule(t *testing.T) { }, }, }, + "resources": { + Input: &configs.Config{ + Module: &configs.Module{ + ManagedResources: map[string]*configs.Resource{ + "test_res": { + Mode: addrs.ManagedResourceMode, + Name: "test_res", + Type: "test_type", + Config: &hclsyntax.Body{ + Attributes: map[string]*hclsyntax.Attribute{}, + }, + Provider: providerAddr, + }, + }, + DataResources: map[string]*configs.Resource{ + "test_data": { + Mode: addrs.DataResourceMode, + Name: "test_data", + Type: "test_type", + Config: &hclsyntax.Body{ + Attributes: map[string]*hclsyntax.Attribute{}, + }, + Provider: providerAddr, + }, + }, + EphemeralResources: map[string]*configs.Resource{ + "test_ephemeral": { + Mode: addrs.EphemeralResourceMode, + Name: "test_ephemeral", + Type: "test_type", + Config: &hclsyntax.Body{ + Attributes: map[string]*hclsyntax.Attribute{}, + }, + Provider: providerAddr, + }, + }, + }, + }, + Schemas: &tofu.Schemas{ + Providers: map[addrs.Provider]providers.ProviderSchema{ + providerAddr: { + ResourceTypes: resSchema, + EphemeralResources: resSchema, + DataSources: resSchema, + }, + }, + }, + Want: module{ + Outputs: map[string]output{}, + ModuleCalls: map[string]moduleCall{}, + Resources: []resource{ + { + Address: "test_type.test_res", + Mode: "managed", + Type: "test_type", + Name: "test_res", + ProviderConfigKey: "test", + SchemaVersion: ptrTo[uint64](0), + Provisioners: nil, + Expressions: make(map[string]any), + }, + { + Address: "data.test_type.test_data", + Mode: "data", + Type: "test_type", + Name: "test_data", + ProviderConfigKey: "test", + SchemaVersion: ptrTo[uint64](0), + Provisioners: nil, + Expressions: make(map[string]any), + }, + { + Address: "ephemeral.test_type.test_ephemeral", + Mode: "ephemeral", + Type: "test_type", + Name: "test_ephemeral", + ProviderConfigKey: "test", + SchemaVersion: ptrTo[uint64](0), + Provisioners: nil, + Expressions: make(map[string]any), + }, + }, + }, + }, // TODO: More test cases covering things other than input variables. // (For now the other details are mainly tested in package command, // as part of the tests for "tofu show".) @@ -275,3 +375,12 @@ func TestMarshalModule(t *testing.T) { }) } } + +// ptrTo is a helper to compensate for the fact that Go doesn't allow +// using the '&' operator unless the operand is directly addressable. +// +// Instead then, this function returns a pointer to a copy of the given +// value. +func ptrTo[T any](v T) *T { + return &v +} diff --git a/internal/command/jsonentities/change.go b/internal/command/jsonentities/change.go index 74615db5f4..8fe19e1052 100644 --- a/internal/command/jsonentities/change.go +++ b/internal/command/jsonentities/change.go @@ -73,6 +73,7 @@ const ( ActionDelete ChangeAction = "delete" ActionImport ChangeAction = "import" ActionForget ChangeAction = "remove" + ActionOpen ChangeAction = "open" ) func ParseChangeAction(action plans.Action) ChangeAction { @@ -91,6 +92,9 @@ func ParseChangeAction(action plans.Action) ChangeAction { return ActionDelete case plans.Forget: return ActionForget + case plans.Open: + return ActionOpen + // NOTE: Renew and Close missing on purpose since those are not meant to be stored default: return ActionNoOp } diff --git a/internal/command/jsonformat/plan.go b/internal/command/jsonformat/plan.go index 8ade742d11..8b6f16ecd8 100644 --- a/internal/command/jsonformat/plan.go +++ b/internal/command/jsonformat/plan.go @@ -43,6 +43,8 @@ func (plan Plan) getSchema(change jsonplan.ResourceChange) *jsonprovider.Schema return plan.ProviderSchemas[change.ProviderName].ResourceSchemas[change.Type] case jsonstate.DataResourceMode: return plan.ProviderSchemas[change.ProviderName].DataSourceSchemas[change.Type] + case jsonstate.EphemeralResourceMode: + return plan.ProviderSchemas[change.ProviderName].EphemeralResourceSchemas[change.Type] default: panic("found unrecognized resource mode: " + change.Mode) } @@ -76,6 +78,10 @@ func (plan Plan) renderHuman(renderer Renderer, mode plans.Mode, opts ...plans.Q // Don't render anything for deleted data sources. continue } + if diff.change.Mode == jsonstate.EphemeralResourceMode { + // Do not render ephemeral changes. // TODO ephemeral add e2e test for this + continue + } changes = append(changes, diff) @@ -361,6 +367,10 @@ func renderHumanDiffDrift(renderer Renderer, diffs diffs, mode plans.Mode) bool } func renderHumanDiff(renderer Renderer, diff diff, cause string) (string, bool) { + if diff.change.Mode == jsonstate.EphemeralResourceMode { + // render nothing for ephemeral resources + return "", false + } // Internally, our computed diffs can't tell the difference between a // replace action (eg. CreateThenDestroy, DestroyThenCreate) and a simple @@ -569,6 +579,8 @@ func actionDescription(action plans.Action) string { return " [cyan]<=[reset] read (data resources)" case plans.Forget: return " [red].[reset] forget" + case plans.Open: + panic("ephemeral changes are not meant to be printed") default: panic(fmt.Sprintf("unrecognized change type: %s", action.String())) diff --git a/internal/command/jsonformat/plan_test.go b/internal/command/jsonformat/plan_test.go index eff1515f4e..0deda9504a 100644 --- a/internal/command/jsonformat/plan_test.go +++ b/internal/command/jsonformat/plan_test.go @@ -1069,6 +1069,22 @@ new line`), # (2 unchanged attributes hidden) }`, }, + "open ephemeral": { + Action: plans.Open, + Mode: addrs.EphemeralResourceMode, + Before: cty.ObjectVal(map[string]cty.Value{ + "name": cty.StringVal("name"), + }), + After: cty.ObjectVal(map[string]cty.Value{ + "name": cty.StringVal("name"), + }), + Schema: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "name": {Type: cty.String, Optional: true}, + }, + }, + ExpectedOutput: ``, + }, } runTestCases(t, testCases) @@ -1685,6 +1701,26 @@ func TestResourceChange_JSON(t *testing.T) { ) }`, }, + "ephemeral resource creation": { + Action: plans.Create, + Mode: addrs.EphemeralResourceMode, + Before: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("ad4d04ed-ac40-43fc-ad4f-d2fc89b80793"), + "json_field": cty.StringVal(`{"secret_value": "8f6fb348-949d-4fa3-98a4-da9e66088257"}`), + }), + After: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("ad4d04ed-ac40-43fc-ad4f-d2fc89b80793"), + "json_field": cty.StringVal(`{"secret_value": "f8b90277-7b0b-4f15-9c31-aed5a642b274"}`), + }), + Schema: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": {Type: cty.String, Optional: true, Computed: true}, + "json_field": {Type: cty.String, Optional: true}, + }, + }, + RequiredReplace: cty.NewPathSet(), + ExpectedOutput: ``, + }, } runTestCases(t, testCases) } @@ -2290,6 +2326,74 @@ func TestResourceChange_primitiveSet(t *testing.T) { # (1 unchanged attribute hidden) }`, }, + "fails when ephemeral in the after marks": { + Action: plans.Update, + Mode: addrs.ManagedResourceMode, + Before: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("i-02ae66f368e8518a9"), + "ami": cty.StringVal("ami-STATIC"), + "set_field": cty.NullVal(cty.Set(cty.String)), + }), + After: cty.ObjectVal(map[string]cty.Value{ + "id": cty.UnknownVal(cty.String), + "ami": cty.StringVal("ami-STATIC"), + "set_field": cty.SetVal([]cty.Value{ + cty.StringVal("new-element"), + }), + }), + AfterValMarks: []cty.PathValueMarks{ + { + Path: cty.GetAttrPath("set_field").IndexInt(0), + Marks: map[interface{}]struct{}{ + marks.Ephemeral: {}, + }, + }, + }, + Schema: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": {Type: cty.String, Optional: true, Computed: true}, + "ami": {Type: cty.String, Optional: true}, + "set_field": {Type: cty.Set(cty.String), Optional: true}, + }, + }, + RequiredReplace: cty.NewPathSet(), + ExpectedOutput: ``, + ExpectedErr: fmt.Errorf("test_instance.example: ephemeral marks found at the following paths:\n.set_field[0]"), + }, + "fails when ephemeral in the before marks": { + Action: plans.Update, + Mode: addrs.ManagedResourceMode, + Before: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("i-02ae66f368e8518a9"), + "ami": cty.StringVal("ami-STATIC"), + "set_field": cty.NullVal(cty.Set(cty.String)), + }), + After: cty.ObjectVal(map[string]cty.Value{ + "id": cty.UnknownVal(cty.String), + "ami": cty.StringVal("ami-STATIC"), + "set_field": cty.SetVal([]cty.Value{ + cty.StringVal("new-element"), + }), + }), + BeforeValMarks: []cty.PathValueMarks{ + { + Path: cty.GetAttrPath("set_field").IndexInt(0), + Marks: map[interface{}]struct{}{ + marks.Ephemeral: {}, + }, + }, + }, + Schema: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": {Type: cty.String, Optional: true, Computed: true}, + "ami": {Type: cty.String, Optional: true}, + "set_field": {Type: cty.Set(cty.String), Optional: true}, + }, + }, + RequiredReplace: cty.NewPathSet(), + ExpectedOutput: ``, + ExpectedErr: fmt.Errorf("test_instance.example: ephemeral marks found at the following paths:\n.set_field[0]"), + }, "in-place update - first insertion": { Action: plans.Update, Mode: addrs.ManagedResourceMode, @@ -7172,6 +7276,7 @@ type testCase struct { RequiredReplace cty.PathSet ExpectedOutput string PrevRunAddr addrs.AbsResourceInstance + ExpectedErr error } func runTestCases(t *testing.T, testCases map[string]testCase) { @@ -7253,23 +7358,41 @@ func runTestCases(t *testing.T, testCases map[string]testCase) { Block: tc.Schema, }, }, + EphemeralResources: map[string]providers.Schema{ + src.Addr.Resource.Resource.Type: { + Block: tc.Schema, + }, + }, }, }, } jsonchanges, err := jsonplan.MarshalResourceChanges([]*plans.ResourceInstanceChangeSrc{src}, tfschemas) if err != nil { - t.Errorf("failed to marshal resource changes: %s", err.Error()) - return + if tc.ExpectedErr == nil { + t.Errorf("failed to marshal resource changes.\ngot err:\n%s\nbut no expected err", err) + } else { + gotErr := err.Error() + wantErr := tc.ExpectedErr.Error() + if gotErr != wantErr { + t.Errorf("failed to marshal resource changes.\ngot err:\n%s\nexpected err:\n%s", gotErr, wantErr) + } + } + } else if tc.ExpectedErr != nil { + t.Errorf("failed to marshal resource changes.\nwant err:\n%s\nbut got none", tc.ExpectedErr) } jsonschemas := jsonprovider.MarshalForRenderer(tfschemas) - change := structured.FromJsonChange(jsonchanges[0].Change, attribute_path.AlwaysMatcher()) - renderer := Renderer{Colorize: color} - diff := diff{ - change: jsonchanges[0], - diff: differ.ComputeDiffForBlock(change, jsonschemas[jsonchanges[0].ProviderName].ResourceSchemas[jsonchanges[0].Type].Block), + + var output string + if len(jsonchanges) > 0 { + change := structured.FromJsonChange(jsonchanges[0].Change, attribute_path.AlwaysMatcher()) + renderer := Renderer{Colorize: color} + diff := diff{ + change: jsonchanges[0], + diff: differ.ComputeDiffForBlock(change, jsonschemas[jsonchanges[0].ProviderName].ResourceSchemas[jsonchanges[0].Type].Block), + } + output, _ = renderHumanDiff(renderer, diff, proposedChange) } - output, _ := renderHumanDiff(renderer, diff, proposedChange) if diff := cmp.Diff(output, tc.ExpectedOutput); diff != "" { t.Errorf("wrong output\nexpected:\n%s\nactual:\n%s\ndiff:\n%s\n", tc.ExpectedOutput, output, diff) } diff --git a/internal/command/jsonformat/state.go b/internal/command/jsonformat/state.go index af6da0f007..2920be434e 100644 --- a/internal/command/jsonformat/state.go +++ b/internal/command/jsonformat/state.go @@ -6,6 +6,7 @@ package jsonformat import ( + "fmt" "sort" ctyjson "github.com/zclconf/go-cty/cty/json" @@ -36,6 +37,8 @@ func (state State) GetSchema(resource jsonstate.Resource) *jsonprovider.Schema { return state.ProviderSchemas[resource.ProviderName].ResourceSchemas[resource.Type] case jsonstate.DataResourceMode: return state.ProviderSchemas[resource.ProviderName].DataSourceSchemas[resource.Type] + case jsonstate.EphemeralResourceMode: + panic(fmt.Errorf("ephemeral resources are not meant to be stored in the state file but schema for ephemeral %s.%s has been requested", resource.Type, resource.Name)) default: panic("found unrecognized resource mode: " + resource.Mode) } @@ -74,6 +77,8 @@ func (state State) renderHumanStateModule(renderer Renderer, module jsonstate.Mo case jsonstate.DataResourceMode: change := structured.FromJsonResource(resource) renderer.Streams.Printf("data %q %q %s", resource.Type, resource.Name, differ.ComputeDiffForBlock(change, schema.Block).RenderHuman(0, opts)) + case jsonstate.EphemeralResourceMode: + panic(fmt.Errorf("ephemeral resource %s %s not allowed to be stored in the state", resource.Type, resource.Name)) default: panic("found unrecognized resource mode: " + resource.Mode) } diff --git a/internal/command/jsonplan/plan.go b/internal/command/jsonplan/plan.go index 482bda0aaf..c1ef3a0270 100644 --- a/internal/command/jsonplan/plan.go +++ b/internal/command/jsonplan/plan.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/opentofu/opentofu/internal/lang/marks" "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" @@ -396,6 +397,16 @@ func MarshalResourceChanges(resources []*plans.ResourceInstanceChangeSrc, schema r.PreviousAddress = rc.PrevRunAddr.String() } + if addr.Resource.Resource.Mode == addrs.EphemeralResourceMode { + // We need to write ephemeral resources to the plan file to be able to build + // the apply graph on `tofu apply `. + // The DiffTransformer needs the changes from the plan to be able to generate + // executable resource instance graph nodes, so we are adding the ephemeral resources too. + // Even though we are writing these, the actual values of the ephemeral *must not* + // be written to the plan so nullify these. + rc.ChangeSrc.Before = nil + rc.ChangeSrc.After = nil + } dataSource := addr.Resource.Resource.Mode == addrs.DataResourceMode // We create "delete" actions for data resources so we can clean up // their entries in state, but this is an implementation detail that @@ -431,11 +442,14 @@ func MarshalResourceChanges(resources []*plans.ResourceInstanceChangeSrc, schema if err != nil { return nil, err } - marks := rc.BeforeValMarks - if schema.ContainsSensitive() { - marks = append(marks, schema.ValueMarks(changeV.Before, nil)...) + valMarks := rc.BeforeValMarks + if schema.ContainsMarks() { + valMarks = append(valMarks, schema.ValueMarks(changeV.Before, nil)...) } - bs := jsonstate.SensitiveAsBoolWithPathValueMarks(changeV.Before, marks) + if err := ensureEphemeralMarksAreValid(addr, valMarks); err != nil { + return nil, err + } + bs := jsonstate.SensitiveAsBoolWithPathValueMarks(changeV.Before, valMarks) beforeSensitive, err = ctyjson.Marshal(bs, bs.Type()) if err != nil { return nil, err @@ -460,11 +474,14 @@ func MarshalResourceChanges(resources []*plans.ResourceInstanceChangeSrc, schema } afterUnknown = unknownAsBool(changeV.After) } - marks := rc.AfterValMarks - if schema.ContainsSensitive() { - marks = append(marks, schema.ValueMarks(changeV.After, nil)...) + valMarks := rc.AfterValMarks + if schema.ContainsMarks() { + valMarks = append(valMarks, schema.ValueMarks(changeV.After, nil)...) } - as := jsonstate.SensitiveAsBoolWithPathValueMarks(changeV.After, marks) + if err := ensureEphemeralMarksAreValid(addr, valMarks); err != nil { + return nil, err + } + as := jsonstate.SensitiveAsBoolWithPathValueMarks(changeV.After, valMarks) afterSensitive, err = ctyjson.Marshal(as, as.Type()) if err != nil { return nil, err @@ -514,6 +531,8 @@ func MarshalResourceChanges(resources []*plans.ResourceInstanceChangeSrc, schema r.Mode = jsonstate.ManagedResourceMode case addrs.DataResourceMode: r.Mode = jsonstate.DataResourceMode + case addrs.EphemeralResourceMode: + r.Mode = jsonstate.EphemeralResourceMode default: return nil, fmt.Errorf("resource %s has an unsupported mode %s", r.Address, addr.Resource.Resource.Mode.String()) } @@ -562,6 +581,18 @@ func MarshalResourceChanges(resources []*plans.ResourceInstanceChangeSrc, schema return ret, nil } +func ensureEphemeralMarksAreValid(addr addrs.AbsResourceInstance, valMarks []cty.PathValueMarks) error { + // ephemeral resources will have the ephemeral mark at the root of the value, got from schema.ValueMarks + // so we don't want to error for those particular ones + if addr.Resource.Resource.Mode == addrs.EphemeralResourceMode { + return nil + } + if err := marks.EnsureNoEphemeralMarks(valMarks); err != nil { + return fmt.Errorf("%s: %w", addr, err) + } + return nil +} + // GenerateChange is used to receive two values and calculate the difference // between them in order to return a Change struct func GenerateChange(beforeVal, afterVal cty.Value) (*Change, error) { @@ -868,6 +899,8 @@ func actionString(action string) []string { return []string{"delete", "create"} case "Forget": return []string{"forget"} + case "Open": + return []string{"open"} default: return []string{action} } @@ -899,6 +932,8 @@ func UnmarshalActions(actions []string) plans.Action { return plans.NoOp case "forget": return plans.Forget + case "open": + return plans.Open } } diff --git a/internal/command/jsonplan/values.go b/internal/command/jsonplan/values.go index ffe343d735..9ce759b1a8 100644 --- a/internal/command/jsonplan/values.go +++ b/internal/command/jsonplan/values.go @@ -205,6 +205,8 @@ func marshalPlanResources(changeMap map[string]*plans.ResourceInstanceChangeSrc, resource.Mode = "managed" case addrs.DataResourceMode: resource.Mode = "data" + case addrs.EphemeralResourceMode: + resource.Mode = "ephemeral" default: return nil, fmt.Errorf("resource %s has an unsupported mode %s", r.Addr.String(), diff --git a/internal/command/jsonprovider/provider.go b/internal/command/jsonprovider/provider.go index f4b0de832c..0e24806e7b 100644 --- a/internal/command/jsonprovider/provider.go +++ b/internal/command/jsonprovider/provider.go @@ -24,10 +24,11 @@ type Providers struct { } type Provider struct { - Provider *Schema `json:"provider,omitempty"` - ResourceSchemas map[string]*Schema `json:"resource_schemas,omitempty"` - DataSourceSchemas map[string]*Schema `json:"data_source_schemas,omitempty"` - Functions map[string]*Function `json:"functions,omitempty"` + Provider *Schema `json:"provider,omitempty"` + ResourceSchemas map[string]*Schema `json:"resource_schemas,omitempty"` + DataSourceSchemas map[string]*Schema `json:"data_source_schemas,omitempty"` + EphemeralResourceSchemas map[string]*Schema `json:"ephemeral_resource_schemas,omitempty"` + Functions map[string]*Function `json:"functions,omitempty"` } func newProviders() *Providers { @@ -59,9 +60,10 @@ func Marshal(s *tofu.Schemas) ([]byte, error) { func marshalProvider(tps providers.ProviderSchema) *Provider { return &Provider{ - Provider: marshalSchema(tps.Provider), - ResourceSchemas: marshalSchemas(tps.ResourceTypes), - DataSourceSchemas: marshalSchemas(tps.DataSources), - Functions: marshalFunctions(tps.Functions), + Provider: marshalSchema(tps.Provider), + ResourceSchemas: marshalSchemas(tps.ResourceTypes), + DataSourceSchemas: marshalSchemas(tps.DataSources), + EphemeralResourceSchemas: marshalSchemas(tps.EphemeralResources), + Functions: marshalFunctions(tps.Functions), } } diff --git a/internal/command/jsonprovider/provider_test.go b/internal/command/jsonprovider/provider_test.go index b73aeb6a9e..181a08eadd 100644 --- a/internal/command/jsonprovider/provider_test.go +++ b/internal/command/jsonprovider/provider_test.go @@ -25,10 +25,11 @@ func TestMarshalProvider(t *testing.T) { { providers.ProviderSchema{}, &Provider{ - Provider: &Schema{}, - ResourceSchemas: map[string]*Schema{}, - DataSourceSchemas: map[string]*Schema{}, - Functions: map[string]*Function{}, + Provider: &Schema{}, + ResourceSchemas: map[string]*Schema{}, + DataSourceSchemas: map[string]*Schema{}, + EphemeralResourceSchemas: map[string]*Schema{}, + Functions: map[string]*Function{}, }, }, { @@ -147,6 +148,47 @@ func TestMarshalProvider(t *testing.T) { }, }, }, + EphemeralResourceSchemas: map[string]*Schema{ + "test_ephemeral_resource": { + Version: 4, + Block: &Block{ + Attributes: map[string]*Attribute{ + "id": { + AttributeType: json.RawMessage(`"string"`), + Optional: true, + Computed: true, + DescriptionKind: "plain", + }, + "secret": { + AttributeType: json.RawMessage(`"string"`), + Optional: true, + DescriptionKind: "plain", + }, + }, + BlockTypes: map[string]*BlockType{ + "notes": { + Block: &Block{ + Attributes: map[string]*Attribute{ + "secret1": { + AttributeType: json.RawMessage(`"string"`), + Optional: true, + DescriptionKind: "plain", + }, + "secret2": { + AttributeType: json.RawMessage(`"string"`), + Optional: true, + DescriptionKind: "plain", + }, + }, + DescriptionKind: "plain", + }, + NestingMode: "list", + }, + }, + DescriptionKind: "plain", + }, + }, + }, Functions: map[string]*Function{}, }, }, @@ -225,6 +267,28 @@ func testProvider() providers.ProviderSchema { }, }, }, + EphemeralResources: map[string]providers.Schema{ + "test_ephemeral_resource": { + Version: 4, + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": {Type: cty.String, Optional: true, Computed: true}, + "secret": {Type: cty.String, Optional: true}, + }, + BlockTypes: map[string]*configschema.NestedBlock{ + "notes": { + Nesting: configschema.NestingList, + Block: configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "secret1": {Type: cty.String, Optional: true}, + "secret2": {Type: cty.String, Optional: true}, + }, + }, + }, + }, + }, + }, + }, Functions: map[string]providers.FunctionSpec{}, } } diff --git a/internal/command/jsonstate/state.go b/internal/command/jsonstate/state.go index 6852e46e17..fb5e52fb9b 100644 --- a/internal/command/jsonstate/state.go +++ b/internal/command/jsonstate/state.go @@ -27,8 +27,9 @@ const ( // consuming parser. FormatVersion = "1.0" - ManagedResourceMode = "managed" - DataResourceMode = "data" + ManagedResourceMode = "managed" + DataResourceMode = "data" + EphemeralResourceMode = "ephemeral" ) // State is the top-level representation of the json format of a tofu @@ -365,8 +366,9 @@ func marshalResources(resources map[string]*states.Resource, module addrs.Module resAddr := r.Addr.Resource + instAddr := r.Addr.Instance(k) current := Resource{ - Address: r.Addr.Instance(k).String(), + Address: instAddr.String(), Type: resAddr.Type, Name: resAddr.Name, ProviderName: r.ProviderConfig.Provider.String(), @@ -384,6 +386,8 @@ func marshalResources(resources map[string]*states.Resource, module addrs.Module current.Mode = ManagedResourceMode case addrs.DataResourceMode: current.Mode = DataResourceMode + case addrs.EphemeralResourceMode: + return ret, fmt.Errorf("ephemeral resource %q detected in the current state. This is an error in OpenTofu", resAddr.String()) default: return ret, fmt.Errorf("resource %s has an unsupported mode %s", resAddr.String(), @@ -415,11 +419,18 @@ func marshalResources(resources map[string]*states.Resource, module addrs.Module current.AttributeValues = marshalAttributeValues(riObj.Value) - value, marks := riObj.Value.UnmarkDeepWithPaths() - if schema.ContainsSensitive() { - marks = append(marks, schema.ValueMarks(value, nil)...) + value, valMarks := riObj.Value.UnmarkDeepWithPaths() + if schema.ContainsMarks() { + valMarks = append(valMarks, schema.ValueMarks(value, nil)...) } - s := SensitiveAsBoolWithPathValueMarks(value, marks) + // NOTE: Even though at this point, the resources that are processed here + // should have no ephemeral mark, we want to validate that before having + // these written to the state. + if err := marks.EnsureNoEphemeralMarks(valMarks); err != nil { + return nil, fmt.Errorf("%s: %w", instAddr, err) + } + + s := SensitiveAsBoolWithPathValueMarks(value, valMarks) v, err := ctyjson.Marshal(s, s.Type()) if err != nil { return nil, err @@ -466,11 +477,17 @@ func marshalResources(resources map[string]*states.Resource, module addrs.Module deposed.AttributeValues = marshalAttributeValues(riObj.Value) - value, marks := riObj.Value.UnmarkDeepWithPaths() - if schema.ContainsSensitive() { - marks = append(marks, schema.ValueMarks(value, nil)...) + value, valMarks := riObj.Value.UnmarkDeepWithPaths() + if schema.ContainsMarks() { + valMarks = append(valMarks, schema.ValueMarks(value, nil)...) } - s := SensitiveAsBool(value.MarkWithPaths(marks)) + // NOTE: Even though at this point, the resources that are processed here + // should have no ephemeral mark, we want to validate that before having + // these written to the state. + if err := marks.EnsureNoEphemeralMarks(valMarks); err != nil { + return nil, fmt.Errorf("%s: %w", instAddr, err) + } + s := SensitiveAsBool(value.MarkWithPaths(valMarks)) v, err := ctyjson.Marshal(s, s.Type()) if err != nil { return nil, err diff --git a/internal/command/jsonstate/state_test.go b/internal/command/jsonstate/state_test.go index a4a6c789d5..0fef65306d 100644 --- a/internal/command/jsonstate/state_test.go +++ b/internal/command/jsonstate/state_test.go @@ -8,6 +8,7 @@ package jsonstate import ( "encoding/json" "reflect" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -203,13 +204,13 @@ func TestMarshalResources(t *testing.T) { Resources map[string]*states.Resource Schemas *tofu.Schemas Want []Resource - Err bool + ErrMsg string }{ "nil": { nil, nil, nil, - false, + "", }, "single resource": { map[string]*states.Resource{ @@ -251,7 +252,49 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage("{\"foozles\":true}"), }, }, - false, + "", + }, + "single data source": { + map[string]*states.Resource{ + "test_thing.baz": { + Addr: addrs.AbsResource{ + Resource: addrs.Resource{ + Mode: addrs.DataResourceMode, + Type: "test_thing", + Name: "bar", + }, + }, + Instances: map[addrs.InstanceKey]*states.ResourceInstance{ + addrs.NoKey: { + Current: &states.ResourceInstanceObjectSrc{ + Status: states.ObjectReady, + AttrsJSON: []byte(`{"foo":"baz"}`), + }, + }, + }, + ProviderConfig: addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("test"), + Module: addrs.RootModule, + }, + }, + }, + testSchemas(), + []Resource{ + { + Address: "data.test_thing.bar", + Mode: "data", + Type: "test_thing", + Name: "bar", + Index: nil, + ProviderName: "registry.opentofu.org/hashicorp/test", + AttributeValues: AttributeValues{ + "foo": json.RawMessage(`"baz"`), + "bar": json.RawMessage(`null`), + }, + SensitiveValues: json.RawMessage("{\"bar\":true}"), + }, + }, + "", }, "single resource_with_sensitive": { map[string]*states.Resource{ @@ -293,9 +336,9 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage("{\"foozles\":true}"), }, }, - false, + "", }, - "resource with marks": { + "resource with sensitive marks": { map[string]*states.Resource{ "test_thing.bar": { Addr: addrs.AbsResource{ @@ -339,7 +382,39 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage(`{"foozles":true}`), }, }, - false, + "", + }, + "resource with ephemeral": { + map[string]*states.Resource{ + "test_thing.bar": { + Addr: addrs.AbsResource{ + Resource: addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "test_thing", + Name: "bar", + }, + }, + Instances: map[addrs.InstanceKey]*states.ResourceInstance{ + addrs.NoKey: { + Current: &states.ResourceInstanceObjectSrc{ + Status: states.ObjectReady, + AttrsJSON: []byte(`{"foozles":"confuzles"}`), + AttrSensitivePaths: []cty.PathValueMarks{{ + Path: cty.Path{cty.GetAttrStep{Name: "foozles"}}, + Marks: cty.NewValueMarks(marks.Ephemeral)}, + }, + }, + }, + }, + ProviderConfig: addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("test"), + Module: addrs.RootModule, + }, + }, + }, + testSchemas(), + nil, + "test_thing.bar: ephemeral marks found at the following paths:\n.foozles", }, "single resource wrong schema": { map[string]*states.Resource{ @@ -368,7 +443,7 @@ func TestMarshalResources(t *testing.T) { }, testSchemas(), nil, - true, + "schema version 1 for test_thing.bar in state does not match version 0 from the provider", }, "resource with count": { map[string]*states.Resource{ @@ -410,7 +485,7 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage("{\"foozles\":true}"), }, }, - false, + "", }, "resource with for_each": { map[string]*states.Resource{ @@ -452,7 +527,7 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage("{\"foozles\":true}"), }, }, - false, + "", }, "deposed resource": { map[string]*states.Resource{ @@ -497,7 +572,7 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage("{\"foozles\":true}"), }, }, - false, + "", }, "deposed and current resource": { map[string]*states.Resource{ @@ -559,7 +634,7 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage("{\"foozles\":true}"), }, }, - false, + "", }, "resource with marked map attr": { map[string]*states.Resource{ @@ -604,17 +679,48 @@ func TestMarshalResources(t *testing.T) { SensitiveValues: json.RawMessage(`{"data":true}`), }, }, - false, + ``, + }, + "single ephemeral resource": { + map[string]*states.Resource{ + "test_thing.baz": { + Addr: addrs.AbsResource{ + Resource: addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_thing", + Name: "bar", + }, + }, + Instances: map[addrs.InstanceKey]*states.ResourceInstance{ + addrs.NoKey: { + Current: &states.ResourceInstanceObjectSrc{ + Status: states.ObjectReady, + AttrsJSON: []byte(`{"foo":"baz"}`), + }, + }, + }, + ProviderConfig: addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("test"), + Module: addrs.RootModule, + }, + }, + }, + testSchemas(), + nil, + `ephemeral resource "ephemeral.test_thing.bar" detected in the current state. This is an error in OpenTofu`, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { got, err := marshalResources(test.Resources, addrs.RootModuleInstance, test.Schemas) - if test.Err { + if test.ErrMsg != "" { if err == nil { t.Fatal("succeeded; want error") } + if !strings.Contains(err.Error(), test.ErrMsg) { + t.Fatalf("expected msg %q in error %q", test.ErrMsg, err.Error()) + } return } else if err != nil { t.Fatalf("unexpected error: %s", err) @@ -862,6 +968,26 @@ func testSchemas() *tofu.Schemas { }, }, }, + DataSources: map[string]providers.Schema{ + "test_thing": { + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "foo": {Type: cty.String, Optional: true, Computed: true}, + "bar": {Type: cty.String, Optional: true, Sensitive: true}, + }, + }, + }, + }, + EphemeralResources: map[string]providers.Schema{ + "test_thing": { + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "foo": {Type: cty.String, Optional: true, Computed: true}, + "bar": {Type: cty.String, Optional: true, Sensitive: true}, + }, + }, + }, + }, }, }, } diff --git a/internal/command/state_mv.go b/internal/command/state_mv.go index 3b21a3eb31..d3c9a2ae4e 100644 --- a/internal/command/state_mv.go +++ b/internal/command/state_mv.go @@ -493,8 +493,19 @@ func (c *StateMvCommand) sourceObjectAddrs(state *states.State, matched addrs.Ta func (c *StateMvCommand) validateResourceMove(addrFrom, addrTo addrs.AbsResource) tfdiags.Diagnostics { const msgInvalidRequest = "Invalid state move request" - var diags tfdiags.Diagnostics + + if addrFrom.Resource.Mode == addrs.EphemeralResourceMode || addrTo.Resource.Mode == addrs.EphemeralResourceMode { + diags = diags.Append( + tfdiags.Sourceless( + tfdiags.Error, + msgInvalidRequest, + "Ephemeral resources cannot be used as sources or targets for the move action. Just update your configuration accordingly.", + ), + ) + return diags + } + if addrFrom.Resource.Mode != addrTo.Resource.Mode { switch addrFrom.Resource.Mode { case addrs.ManagedResourceMode: @@ -509,6 +520,7 @@ func (c *StateMvCommand) validateResourceMove(addrFrom, addrTo addrs.AbsResource msgInvalidRequest, fmt.Sprintf("Cannot move %s to %s: a data resource can be moved only to another data resource address.", addrFrom, addrTo), )) + // NOTE: No need for the ephemeral resource in this switch block since it is handled at the top of the method. default: // In case a new mode is added in future, this unhelpful error is better than nothing. diags = diags.Append(tfdiags.Sourceless( diff --git a/internal/command/state_mv_test.go b/internal/command/state_mv_test.go index af7957e126..1fec06b678 100644 --- a/internal/command/state_mv_test.go +++ b/internal/command/state_mv_test.go @@ -14,6 +14,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/mitchellh/cli" + "github.com/opentofu/opentofu/internal/tfdiags" "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/states" @@ -1819,6 +1820,107 @@ func TestStateMv_checkRequiredVersion(t *testing.T) { } } +func TestValidateResourceMove(t *testing.T) { + var ( + c = &StateMvCommand{} + + managedRes = addrs.AbsResource{Resource: addrs.Resource{Mode: addrs.ManagedResourceMode, Type: "test_type", Name: "test_name"}} + dataRes = addrs.AbsResource{Resource: addrs.Resource{Mode: addrs.DataResourceMode, Type: "test_type", Name: "test_name"}} + ephemeralRes = addrs.AbsResource{Resource: addrs.Resource{Mode: addrs.EphemeralResourceMode, Type: "test_type", Name: "test_name"}} + ) + + tests := map[string]struct { + src, target addrs.AbsResource + wantDiags tfdiags.Diagnostics + }{ + "resource to resource": { + managedRes, + managedRes, + tfdiags.Diagnostics{}, + }, + "data to data": { + dataRes, + dataRes, + tfdiags.Diagnostics{}, + }, + "ephemeral to ephemeral": { + ephemeralRes, + ephemeralRes, + tfdiags.Diagnostics{tfdiags.Sourceless( + tfdiags.Error, + "Invalid state move request", + "Ephemeral resources cannot be used as sources or targets for the move action. Just update your configuration accordingly.", + )}, + }, + "resource to data": { + managedRes, + dataRes, + tfdiags.Diagnostics{tfdiags.Sourceless( + tfdiags.Error, + "Invalid state move request", + fmt.Sprintf("Cannot move %s to %s: a managed resource can be moved only to another managed resource address.", managedRes, dataRes), + )}, + }, + "resource to ephemeral": { + managedRes, + ephemeralRes, + tfdiags.Diagnostics{tfdiags.Sourceless( + tfdiags.Error, + "Invalid state move request", + "Ephemeral resources cannot be used as sources or targets for the move action. Just update your configuration accordingly.", + )}, + }, + "data to resource": { + dataRes, + managedRes, + tfdiags.Diagnostics{tfdiags.Sourceless( + tfdiags.Error, + "Invalid state move request", + fmt.Sprintf("Cannot move %s to %s: a data resource can be moved only to another data resource address.", dataRes, managedRes), + )}, + }, + "data to ephemeral": { + dataRes, + ephemeralRes, + tfdiags.Diagnostics{tfdiags.Sourceless( + tfdiags.Error, + "Invalid state move request", + "Ephemeral resources cannot be used as sources or targets for the move action. Just update your configuration accordingly.", + )}, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + diags := c.validateResourceMove(tt.src, tt.target) + + if got, want := len(diags), len(tt.wantDiags); got != want { + t.Fatalf("expected to have exactly %d diagnostic(s). got: %d", want, got) + } + for i, wantDiag := range tt.wantDiags { + sameDiagnostic(t, diags[i], wantDiag) + } + }) + } +} + +func sameDiagnostic(t *testing.T, gotD, wantD tfdiags.Diagnostic) { + if got, want := gotD.Severity(), wantD.Severity(); got != want { + t.Errorf("wrong severity. got %q; want %q", got, want) + } + if got, want := gotD.Description().Address, wantD.Description().Address; got != want { + t.Errorf("wrong description. got %q; want %q", got, want) + } + if got, want := gotD.Description().Detail, wantD.Description().Detail; got != want { + t.Errorf("wrong detail. got %q; want %q", got, want) + } + if got, want := gotD.Description().Summary, wantD.Description().Summary; got != want { + t.Errorf("wrong summary. got %q; want %q", got, want) + } + if got, want := gotD.ExtraInfo(), wantD.ExtraInfo(); got != want { + t.Errorf("wrong extra info. got %q; want %q", got, want) + } +} + const testStateMvOutputOriginal = ` test_instance.baz: ID = foo diff --git a/internal/command/testing/test_provider.go b/internal/command/testing/test_provider.go index f2edc57f7b..72dcc6f846 100644 --- a/internal/command/testing/test_provider.go +++ b/internal/command/testing/test_provider.go @@ -65,6 +65,7 @@ var ( }, }, }, + // TODO ephemeral - when implementing testing support for ephemeral resources, consider configuring ephemeral schema here } ) diff --git a/internal/command/views/hook_count.go b/internal/command/views/hook_count.go index d94671e359..165cb40af1 100644 --- a/internal/command/views/hook_count.go +++ b/internal/command/views/hook_count.go @@ -96,8 +96,9 @@ func (h *countHook) PostDiff(addr addrs.AbsResourceInstance, gen states.Generati h.Lock() defer h.Unlock() - // We don't count anything for data resources - if addr.Resource.Resource.Mode == addrs.DataResourceMode { + // We don't count anything for data resources and neither for the ephemeral ones. + // TODO ephemeral - test this after the ephemeral resources are introduced entirely + if addr.Resource.Resource.Mode == addrs.DataResourceMode || addr.Resource.Resource.Mode == addrs.EphemeralResourceMode { return tofu.HookActionContinue, nil } diff --git a/internal/command/views/hook_count_test.go b/internal/command/views/hook_count_test.go index 10aff2b405..51c681668d 100644 --- a/internal/command/views/hook_count_test.go +++ b/internal/command/views/hook_count_test.go @@ -265,6 +265,38 @@ func TestCountHookPostDiff_DataSource(t *testing.T) { } } +func TestCountHookPostDiff_Ephemeral(t *testing.T) { + h := new(countHook) + + resources := map[string]plans.Action{ + "foo": plans.Delete, + "bar": plans.NoOp, + "lorem": plans.Update, + "ipsum": plans.Delete, + } + + for k, a := range resources { + addr := addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_instance", + Name: k, + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance) + + _, _ = h.PostDiff(addr, states.CurrentGen, a, cty.DynamicVal, cty.DynamicVal) + } + + expected := new(countHook) + expected.ToAdd = 0 + expected.ToChange = 0 + expected.ToRemoveAndAdd = 0 + expected.ToRemove = 0 + + if !reflect.DeepEqual(expected, h) { + t.Fatalf("Expected %#v, got %#v instead.", + expected, h) + } +} + func TestCountHookApply_ChangeOnly(t *testing.T) { h := new(countHook) diff --git a/internal/command/views/hook_json.go b/internal/command/views/hook_json.go index 4df1c61fb9..01f1ce8acb 100644 --- a/internal/command/views/hook_json.go +++ b/internal/command/views/hook_json.go @@ -174,3 +174,33 @@ func (h *jsonHook) PostRefresh(addr addrs.AbsResourceInstance, gen states.Genera h.view.Hook(json.NewRefreshComplete(addr, idKey, idValue)) return tofu.HookActionContinue, nil } + +func (h *jsonHook) PreOpen(addr addrs.AbsResourceInstance) (tofu.HookAction, error) { + h.view.Hook(json.NewEphemeralStart(addr, "Opening...")) + return tofu.HookActionContinue, nil +} + +func (h *jsonHook) PostOpen(addr addrs.AbsResourceInstance, _ error) (tofu.HookAction, error) { + h.view.Hook(json.NewEphemeralStop(addr, "Open complete")) + return tofu.HookActionContinue, nil +} + +func (h *jsonHook) PreRenew(addr addrs.AbsResourceInstance) (tofu.HookAction, error) { + h.view.Hook(json.NewEphemeralStart(addr, "Renewing...")) + return tofu.HookActionContinue, nil +} + +func (h *jsonHook) PostRenew(addr addrs.AbsResourceInstance, _ error) (tofu.HookAction, error) { + h.view.Hook(json.NewEphemeralStop(addr, "Renew complete")) + return tofu.HookActionContinue, nil +} + +func (h *jsonHook) PreClose(addr addrs.AbsResourceInstance) (tofu.HookAction, error) { + h.view.Hook(json.NewEphemeralStart(addr, "Closing...")) + return tofu.HookActionContinue, nil +} + +func (h *jsonHook) PostClose(addr addrs.AbsResourceInstance, _ error) (tofu.HookAction, error) { + h.view.Hook(json.NewEphemeralStop(addr, "Close complete")) + return tofu.HookActionContinue, nil +} diff --git a/internal/command/views/hook_json_test.go b/internal/command/views/hook_json_test.go index 5446dc5fa3..d7797a41c3 100644 --- a/internal/command/views/hook_json_test.go +++ b/internal/command/views/hook_json_test.go @@ -339,6 +339,190 @@ func TestJSONHook_refresh(t *testing.T) { testJSONViewOutputEquals(t, done(t).Stdout(), want) } +func TestJSONHook_ephemeral(t *testing.T) { + addr := addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_instance", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance) + + cases := []struct { + name string + preF func(hook tofu.Hook) (tofu.HookAction, error) + postF func(hook tofu.Hook) (tofu.HookAction, error) + want []map[string]interface{} + }{ + { + name: "opening", + preF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PreOpen(addr) + }, + postF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PostOpen(addr, nil) + }, + want: []map[string]interface{}{ + { + "@level": "info", + "@message": "ephemeral.test_instance.foo: Opening...", + "@module": "tofu.ui", + "hook": map[string]any{ + "Msg": "Opening...", + "resource": map[string]any{ + "addr": "ephemeral.test_instance.foo", + "implied_provider": "test", + "module": "", + "resource": "ephemeral.test_instance.foo", + "resource_key": nil, + "resource_name": "foo", + "resource_type": "test_instance", + }, + }, + "type": "ephemeral_action_started", + }, + { + "@level": "info", + "@message": "ephemeral.test_instance.foo: Open complete", + "@module": "tofu.ui", + "hook": map[string]any{ + "Msg": "Open complete", + "resource": map[string]any{ + "addr": "ephemeral.test_instance.foo", + "implied_provider": "test", + "module": "", + "resource": "ephemeral.test_instance.foo", + "resource_key": nil, + "resource_name": "foo", + "resource_type": "test_instance", + }, + }, + "type": "ephemeral_action_complete", + }, + }, + }, + { + name: "renewing", + preF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PreRenew(addr) + }, + postF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PostRenew(addr, nil) + }, + want: []map[string]interface{}{ + { + "@level": "info", + "@message": "ephemeral.test_instance.foo: Renewing...", + "@module": "tofu.ui", + "hook": map[string]any{ + "Msg": "Renewing...", + "resource": map[string]any{ + "addr": "ephemeral.test_instance.foo", + "implied_provider": "test", + "module": "", + "resource": "ephemeral.test_instance.foo", + "resource_key": nil, + "resource_name": "foo", + "resource_type": "test_instance", + }, + }, + "type": "ephemeral_action_started", + }, + { + "@level": "info", + "@message": "ephemeral.test_instance.foo: Renew complete", + "@module": "tofu.ui", + "hook": map[string]any{ + "Msg": "Renew complete", + "resource": map[string]any{ + "addr": "ephemeral.test_instance.foo", + "implied_provider": "test", + "module": "", + "resource": "ephemeral.test_instance.foo", + "resource_key": nil, + "resource_name": "foo", + "resource_type": "test_instance", + }, + }, + "type": "ephemeral_action_complete", + }, + }, + }, + { + name: "closing", + preF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PreClose(addr) + }, + postF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PostClose(addr, nil) + }, + want: []map[string]interface{}{ + { + "@level": "info", + "@message": "ephemeral.test_instance.foo: Closing...", + "@module": "tofu.ui", + "hook": map[string]any{ + "Msg": "Closing...", + "resource": map[string]any{ + "addr": "ephemeral.test_instance.foo", + "implied_provider": "test", + "module": "", + "resource": "ephemeral.test_instance.foo", + "resource_key": nil, + "resource_name": "foo", + "resource_type": "test_instance", + }, + }, + "type": "ephemeral_action_started", + }, + { + "@level": "info", + "@message": "ephemeral.test_instance.foo: Close complete", + "@module": "tofu.ui", + "hook": map[string]any{ + "Msg": "Close complete", + "resource": map[string]any{ + "addr": "ephemeral.test_instance.foo", + "implied_provider": "test", + "module": "", + "resource": "ephemeral.test_instance.foo", + "resource_key": nil, + "resource_name": "foo", + "resource_type": "test_instance", + }, + }, + "type": "ephemeral_action_complete", + }, + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + streams, done := terminal.StreamsForTesting(t) + h := newJSONHook(NewJSONView(NewView(streams))) + + action, err := tt.preF(h) + if err != nil { + t.Fatal(err) + } + if action != tofu.HookActionContinue { + t.Fatalf("Expected hook to continue, given: %#v", action) + } + + <-time.After(1100 * time.Millisecond) + + // call postF that will stop the waiting for the action + action, err = tt.postF(h) + if err != nil { + t.Fatal(err) + } + if action != tofu.HookActionContinue { + t.Errorf("Expected hook to continue, given: %#v", action) + } + + testJSONViewOutputEquals(t, done(t).Stdout(), tt.want) + }) + } +} + func testHookReturnValues(t *testing.T, action tofu.HookAction, err error) { t.Helper() diff --git a/internal/command/views/hook_ui.go b/internal/command/views/hook_ui.go index b4828fa769..e6031250a7 100644 --- a/internal/command/views/hook_ui.go +++ b/internal/command/views/hook_ui.go @@ -78,6 +78,8 @@ const ( uiResourceDestroy uiResourceRead uiResourceNoOp + // NOTE: Ephemeral hooks are implemented separately, + // so there are no uiResource entries for Open/Renew/Close actions. ) func (h *UiHook) PreApply(addr addrs.AbsResourceInstance, gen states.Generation, action plans.Action, priorState, plannedNewState cty.Value) (tofu.HookAction, error) { @@ -339,6 +341,120 @@ func (h *UiHook) PostApplyImport(addr addrs.AbsResourceInstance, importing plans return tofu.HookActionContinue, nil } +func (h *UiHook) Deferred(addr addrs.AbsResourceInstance, reason string) (tofu.HookAction, error) { + id := addr.String() + msg := fmt.Sprintf("Deferred due to %s", reason) + + colorized := fmt.Sprintf( + h.view.colorize.Color("[reset][bold]%s: %s"), + id, msg) + + h.println(colorized) + + return tofu.HookActionContinue, nil +} + +func (h *UiHook) PreOpen(addr addrs.AbsResourceInstance) (tofu.HookAction, error) { + return h.preEphemeral(addr, "Opening...", "Still opening...") +} + +func (h *UiHook) PostOpen(addr addrs.AbsResourceInstance, _ error) (tofu.HookAction, error) { + return h.postEphemeral(addr, "Open complete") +} + +func (h *UiHook) PreRenew(addr addrs.AbsResourceInstance) (tofu.HookAction, error) { + return h.preEphemeral(addr, "Renewing...", "Still renewing...") +} + +func (h *UiHook) PostRenew(addr addrs.AbsResourceInstance, _ error) (tofu.HookAction, error) { + return h.postEphemeral(addr, "Renew complete") +} + +func (h *UiHook) PreClose(addr addrs.AbsResourceInstance) (tofu.HookAction, error) { + return h.preEphemeral(addr, "Closing...", "Still closing...") +} + +func (h *UiHook) PostClose(addr addrs.AbsResourceInstance, _ error) (tofu.HookAction, error) { + return h.postEphemeral(addr, "Close complete") +} + +// preEphemeral is the hook implementation that is used before actions like Renew and Close. +// These are specific for ephemeral resources, and we are not using hook methods used for +// the rest of the resource types because these particular 2 operations have no action +// associated. +func (h *UiHook) preEphemeral(addr addrs.AbsResourceInstance, startMsg, stillRunningMsg string) (tofu.HookAction, error) { + dispAddr := addr.String() + + h.println(fmt.Sprintf( + h.view.colorize.Color("[reset][bold]%s: %s[reset]"), + dispAddr, + startMsg, + )) + + key := addr.String() + uiState := uiResourceState{ + DispAddr: key, + Start: time.Now().Round(time.Second), + DoneCh: make(chan struct{}), + done: make(chan struct{}), + } + + h.resourcesLock.Lock() + h.resources[key] = uiState + h.resourcesLock.Unlock() + + go func() { + defer close(uiState.done) + for { + select { + case <-uiState.DoneCh: + return + case <-time.After(h.periodicUiTimer): + // Timer up, show status + } + + h.println(fmt.Sprintf( + h.view.colorize.Color("[reset][bold]%s: %s [%s elapsed][reset]"), + uiState.DispAddr, + stillRunningMsg, + time.Now().Round(time.Second).Sub(uiState.Start), + )) + } + }() + + return tofu.HookActionContinue, nil +} + +// postEphemeral is the hook implementation that is used after actions like Renew and Close. +// These are specific for ephemeral resources, and we are not using hook methods used for +// the rest of the resource types because these particular 2 operations have no action +// associated. +func (h *UiHook) postEphemeral(addr addrs.AbsResourceInstance, msg string) (tofu.HookAction, error) { + id := addr.String() + + h.resourcesLock.Lock() + state := h.resources[id] + if state.DoneCh != nil { + close(state.DoneCh) + } + + delete(h.resources, id) + h.resourcesLock.Unlock() + + addrStr := addr.String() + + colorized := fmt.Sprintf( + h.view.colorize.Color("[reset][bold]%s: %s after %s"), + addrStr, + msg, + time.Now().Round(time.Second).Sub(state.Start), + ) + + h.println(colorized) + + return tofu.HookActionContinue, nil +} + // Wrap calls to the view so that concurrent calls do not interleave println. func (h *UiHook) println(s string) { h.viewLock.Lock() diff --git a/internal/command/views/hook_ui_test.go b/internal/command/views/hook_ui_test.go index 94c259b79b..6d5a92c985 100644 --- a/internal/command/views/hook_ui_test.go +++ b/internal/command/views/hook_ui_test.go @@ -144,6 +144,104 @@ test_instance\.foo: Still modifying... \[id=test, \ds elapsed\] } } +// Test the ephemeral specific hooks +func TestUiHook_ephemeral(t *testing.T) { + addr := addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_instance", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance) + + cases := []struct { + name string + preF func(hook tofu.Hook) (tofu.HookAction, error) + postF func(hook tofu.Hook) (tofu.HookAction, error) + wantOutput string + }{ + { + name: "opening", + preF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PreOpen(addr) + }, + postF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PostOpen(addr, nil) + }, + wantOutput: `ephemeral\.test_instance\.foo: Opening\.\.\. +ephemeral\.test_instance\.foo: Still opening\.\.\. \[\ds elapsed\] +`, + }, + { + name: "renewing", + preF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PreRenew(addr) + }, + postF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PostRenew(addr, nil) + }, + wantOutput: `ephemeral\.test_instance\.foo: Renewing\.\.\. +ephemeral\.test_instance\.foo: Still renewing\.\.\. \[\ds elapsed\] +ephemeral\.test_instance\.foo: Renew complete after \ds +`, + }, + { + name: "closing", + preF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PreClose(addr) + }, + postF: func(hook tofu.Hook) (tofu.HookAction, error) { + return hook.PostClose(addr, nil) + }, + wantOutput: `ephemeral\.test_instance\.foo: Closing\.\.\. +ephemeral\.test_instance\.foo: Still closing\.\.\. \[\ds elapsed\] +ephemeral\.test_instance\.foo: Close complete after \ds +`, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + streams, done := terminal.StreamsForTesting(t) + view := NewView(streams) + h := NewUiHook(view) + h.periodicUiTimer = 1 * time.Second + + action, err := tt.preF(h) + if err != nil { + t.Fatal(err) + } + if action != tofu.HookActionContinue { + t.Fatalf("Expected hook to continue, given: %#v", action) + } + + <-time.After(1100 * time.Millisecond) + + // stop the background writer + uiState := h.resources[addr.String()] + // call postF that will stop the waiting for the action + action, err = tt.postF(h) + if err != nil { + t.Fatal(err) + } + if action != tofu.HookActionContinue { + t.Errorf("Expected hook to continue, given: %#v", action) + } + // wait for the waiting to stop completely + <-uiState.done + + result := done(t) + output := result.Stdout() + if matched, _ := regexp.MatchString(tt.wantOutput, output); !matched { + t.Fatalf("Output didn't match.\nExpected: %q\nGiven: %q", tt.wantOutput, output) + } + + expectedErrOutput := "" + errOutput := result.Stderr() + if errOutput != expectedErrOutput { + t.Fatalf("Error output didn't match.\nExpected: %q\nGiven: %q", expectedErrOutput, errOutput) + } + }) + } +} + // Test the PreApply hook's destroy path, including passing a deposed key as // the gen argument. func TestUiHookPreApply_destroy(t *testing.T) { diff --git a/internal/command/views/json/hook.go b/internal/command/views/json/hook.go index d0563e91ca..398c5bc46e 100644 --- a/internal/command/views/json/hook.go +++ b/internal/command/views/json/hook.go @@ -304,8 +304,56 @@ func NewRefreshComplete(addr addrs.AbsResourceInstance, idKey, idValue string) H } } +// EphemeralStart: triggered by PreOpen, PreRenew and PreClose hooks +type ephemeralStart struct { + Resource jsonentities.ResourceAddr `json:"resource"` + Msg string +} + +var _ Hook = (*ephemeralStart)(nil) + +func (h *ephemeralStart) HookType() MessageType { + return MessageEphemeralActionStart +} + +func (h *ephemeralStart) String() string { + return fmt.Sprintf("%s: %s", h.Resource.Addr, h.Msg) +} + +func NewEphemeralStart(addr addrs.AbsResourceInstance, startMsg string) Hook { + return &ephemeralStart{ + Resource: jsonentities.NewResourceAddr(addr), + Msg: startMsg, + } +} + +// EphemeralStop: triggered by PostOpen, PostRenew and PostClose hooks +type ephemeralStop struct { + Resource jsonentities.ResourceAddr `json:"resource"` + Msg string +} + +var _ Hook = (*ephemeralStop)(nil) + +func (h *ephemeralStop) HookType() MessageType { + return MessageEphemeralActionComplete +} + +func (h *ephemeralStop) String() string { + return fmt.Sprintf("%s: %s", h.Resource.Addr, h.Msg) +} + +func NewEphemeralStop(addr addrs.AbsResourceInstance, startMsg string) Hook { + return &ephemeralStop{ + Resource: jsonentities.NewResourceAddr(addr), + Msg: startMsg, + } +} + // Convert the subset of plans.Action values we expect to receive into a // present-tense verb for the applyStart hook message. +// +// NOTE: Open, Renew and Close missing on purpose since those have their own dedicated hooks. func startActionVerb(action plans.Action) string { switch action { case plans.Create: @@ -334,6 +382,8 @@ func startActionVerb(action plans.Action) string { // Convert the subset of plans.Action values we expect to receive into a // present-tense verb for the applyProgress hook message. This will be // prefixed with "Still ", so it is lower-case. +// +// NOTE: Open, Renew and Close missing on purpose since those have their own dedicated hooks. func progressActionVerb(action plans.Action) string { switch action { case plans.Create: @@ -362,6 +412,8 @@ func progressActionVerb(action plans.Action) string { // Convert the subset of plans.Action values we expect to receive into a // noun for the applyComplete and applyErrored hook messages. This will be // combined into a phrase like "Creation complete after 1m4s". +// +// NOTE: Open, Renew and Close missing on purpose since those have their own dedicated hooks. func actionNoun(action plans.Action) string { switch action { case plans.Create: diff --git a/internal/command/views/json/message_types.go b/internal/command/views/json/message_types.go index 5eccdc63a5..d17af97243 100644 --- a/internal/command/views/json/message_types.go +++ b/internal/command/views/json/message_types.go @@ -20,16 +20,18 @@ const ( MessageOutputs MessageType = "outputs" // Hook-driven messages - MessageApplyStart MessageType = "apply_start" - MessageApplyProgress MessageType = "apply_progress" - MessageApplyComplete MessageType = "apply_complete" - MessageApplyErrored MessageType = "apply_errored" - MessageProvisionStart MessageType = "provision_start" - MessageProvisionProgress MessageType = "provision_progress" - MessageProvisionComplete MessageType = "provision_complete" - MessageProvisionErrored MessageType = "provision_errored" - MessageRefreshStart MessageType = "refresh_start" - MessageRefreshComplete MessageType = "refresh_complete" + MessageApplyStart MessageType = "apply_start" + MessageApplyProgress MessageType = "apply_progress" + MessageApplyComplete MessageType = "apply_complete" + MessageApplyErrored MessageType = "apply_errored" + MessageProvisionStart MessageType = "provision_start" + MessageProvisionProgress MessageType = "provision_progress" + MessageProvisionComplete MessageType = "provision_complete" + MessageProvisionErrored MessageType = "provision_errored" + MessageRefreshStart MessageType = "refresh_start" + MessageRefreshComplete MessageType = "refresh_complete" + MessageEphemeralActionStart MessageType = "ephemeral_action_started" + MessageEphemeralActionComplete MessageType = "ephemeral_action_complete" // Test messages MessageTestAbstract MessageType = "test_abstract" diff --git a/internal/command/views/operation.go b/internal/command/views/operation.go index c602fab7ac..9982052de4 100644 --- a/internal/command/views/operation.go +++ b/internal/command/views/operation.go @@ -277,6 +277,10 @@ func (v *OperationJSON) PlannedChange(change *plans.ResourceInstanceChangeSrc) { // Avoid rendering data sources on deletion return } + if change.Addr.Resource.Resource.Mode == addrs.EphemeralResourceMode { + // Ephemeral changes should not be rendered + return + } v.view.PlannedChange(jsonentities.NewResourceInstanceChange(change)) } diff --git a/internal/command/views/operation_test.go b/internal/command/views/operation_test.go index efd98162cb..104fbadf27 100644 --- a/internal/command/views/operation_test.go +++ b/internal/command/views/operation_test.go @@ -435,6 +435,34 @@ Plan: 1 to add, 0 to change, 0 to destroy. } } +func TestOperation_planWithEphemeral(t *testing.T) { + streams, done := terminal.StreamsForTesting(t) + v := NewOperation(arguments.ViewHuman, true, NewView(streams)) + + plan := testPlanWithEphemeral(t) + schemas := testSchemas() + v.Plan(plan, schemas) + + want := ` +OpenTofu used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + + create + +OpenTofu will perform the following actions: + + # test_resource.foo will be created + + resource "test_resource" "foo" { + + foo = "bar" + + id = (known after apply) + } + +Plan: 1 to add, 0 to change, 0 to destroy. +` + + if got := done(t).Stdout(); got != want { + t.Errorf("unexpected output\ngot:\n%s\nwant:\n%s", got, want) + } +} func TestOperation_planNextStep(t *testing.T) { testCases := map[string]struct { path string @@ -482,6 +510,15 @@ func TestOperationJSON_logs(t *testing.T) { streams, done := terminal.StreamsForTesting(t) v := &OperationJSON{view: NewJSONView(NewView(streams))} + // Added an ephemeral resource change to double-check that it's not + // shown. + v.PlannedChange(&plans.ResourceInstanceChangeSrc{ + Addr: addrs.AbsResourceInstance{ + Resource: addrs.ResourceInstance{ + Resource: addrs.Resource{Mode: addrs.EphemeralResourceMode}, + }, + }, + }) v.Cancelled(plans.NormalMode) v.Cancelled(plans.DestroyMode) v.Stopping() diff --git a/internal/command/views/plan_test.go b/internal/command/views/plan_test.go index 7a47a31666..c189cd347f 100644 --- a/internal/command/views/plan_test.go +++ b/internal/command/views/plan_test.go @@ -133,6 +133,45 @@ func testPlanWithDatasource(t *testing.T) *plans.Plan { return plan } +func testPlanWithEphemeral(t *testing.T) *plans.Plan { + plan := testPlan(t) + + addr := addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_ephemeral_resource", + Name: "bar", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance) + + ephemeralVal := cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("C6743020-40BD-4591-81E6-CD08494341D3"), + "foo": cty.StringVal("baz"), + }) + priorValRaw, err := plans.NewDynamicValue(cty.NullVal(ephemeralVal.Type()), ephemeralVal.Type()) + if err != nil { + t.Fatal(err) + } + plannedValRaw, err := plans.NewDynamicValue(ephemeralVal, ephemeralVal.Type()) + if err != nil { + t.Fatal(err) + } + + plan.Changes.SyncWrapper().AppendResourceInstanceChange(&plans.ResourceInstanceChangeSrc{ + Addr: addr, + PrevRunAddr: addr, + ProviderAddr: addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("test"), + Module: addrs.RootModule, + }, + ChangeSrc: plans.ChangeSrc{ + Action: plans.Open, + Before: priorValRaw, + After: plannedValRaw, + }, + }) + + return plan +} + func testSchemas() *tofu.Schemas { provider := testProvider() return &tofu.Schemas{ @@ -178,5 +217,15 @@ func testProviderSchema() *providers.GetProviderSchemaResponse { }, }, }, + EphemeralResources: map[string]providers.Schema{ + "test_ephemeral_resource": { + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": {Type: cty.String, Required: true}, + "foo": {Type: cty.String, Optional: true}, + }, + }, + }, + }, } } diff --git a/internal/communicator/shared/shared.go b/internal/communicator/shared/shared.go index d8bdd5344a..10804d232e 100644 --- a/internal/communicator/shared/shared.go +++ b/internal/communicator/shared/shared.go @@ -144,6 +144,7 @@ var ConnectionBlockSupersetSchema = &configschema.Block{ Optional: true, }, }, + Ephemeral: true, } // IpFormat formats the IP correctly, so we don't provide IPv6 address in an IPv4 format during node communication. We return the ip parameter as is if it's an IPv4 address or a hostname. diff --git a/internal/configs/config.go b/internal/configs/config.go index 17a585d6b5..82866cee1d 100644 --- a/internal/configs/config.go +++ b/internal/configs/config.go @@ -419,47 +419,9 @@ func (c *Config) addProviderRequirements(reqs getproviders.Requirements, qualifs // Each resource in the configuration creates an *implicit* provider // dependency, though we'll only record it if there isn't already // an explicit dependency on the same provider. - for _, rc := range c.Module.ManagedResources { - fqn := rc.Provider - if _, exists := reqs[fqn]; exists { - // If this is called for a child module, and the provider was added from another implicit reference and not - // from a top level required_provider, we need to collect the reference of this resource as well as implicit provider. - qualifs.AddImplicitProvider(fqn, getproviders.ResourceRef{ - CfgRes: rc.Addr().InModule(c.Path), - Ref: tfdiags.SourceRangeFromHCL(rc.DeclRange), - ProviderAttribute: rc.ProviderConfigRef != nil, - }) - // Explicit dependency already present - continue - } - qualifs.AddImplicitProvider(fqn, getproviders.ResourceRef{ - CfgRes: rc.Addr().InModule(c.Path), - Ref: tfdiags.SourceRangeFromHCL(rc.DeclRange), - ProviderAttribute: rc.ProviderConfigRef != nil, - }) - reqs[fqn] = nil - } - for _, rc := range c.Module.DataResources { - fqn := rc.Provider - if _, exists := reqs[fqn]; exists { - // If this is called for a child module, and the provider was added from another implicit reference and not - // from a top level required_provider, we need to collect the reference of this resource as well as implicit provider. - qualifs.AddImplicitProvider(fqn, getproviders.ResourceRef{ - CfgRes: rc.Addr().InModule(c.Path), - Ref: tfdiags.SourceRangeFromHCL(rc.DeclRange), - ProviderAttribute: rc.ProviderConfigRef != nil, - }) - - // Explicit dependency already present - continue - } - qualifs.AddImplicitProvider(fqn, getproviders.ResourceRef{ - CfgRes: rc.Addr().InModule(c.Path), - Ref: tfdiags.SourceRangeFromHCL(rc.DeclRange), - ProviderAttribute: rc.ProviderConfigRef != nil, - }) - reqs[fqn] = nil - } + c.collectImplicitProviders(c.Module.ManagedResources, reqs, qualifs) + c.collectImplicitProviders(c.Module.DataResources, reqs, qualifs) + c.collectImplicitProviders(c.Module.EphemeralResources, reqs, qualifs) // Import blocks that are generating config may also have a custom provider // meta argument. Like the provider meta argument used in resource blocks, @@ -573,6 +535,31 @@ func (c *Config) addProviderRequirements(reqs getproviders.Requirements, qualifs return diags } +// collectImplicitProviders is checking the provider configuration of each resource. +// For the resources whose required provider is not explicitly configured, an implicit one is collected. +// This is mainly used for enabling warnings when OpenTofu fails to resolve the implicitly generated provider. +func (c *Config) collectImplicitProviders(resources map[string]*Resource, reqs getproviders.Requirements, qualifs *getproviders.ProvidersQualification) { + for _, rc := range resources { + fqn := rc.Provider + if _, exists := reqs[fqn]; exists { + // If this is called for a child module, and the provider was added from another implicit reference and not + // from a top level required_provider, we need to collect the reference of this resource as well as implicit provider. + qualifs.AddImplicitProvider(fqn, getproviders.ResourceRef{ + CfgRes: rc.Addr().InModule(c.Path), + Ref: tfdiags.SourceRangeFromHCL(rc.DeclRange), + ProviderAttribute: rc.ProviderConfigRef != nil, + }) + continue + } + qualifs.AddImplicitProvider(fqn, getproviders.ResourceRef{ + CfgRes: rc.Addr().InModule(c.Path), + Ref: tfdiags.SourceRangeFromHCL(rc.DeclRange), + ProviderAttribute: rc.ProviderConfigRef != nil, + }) + reqs[fqn] = nil + } +} + func (c *Config) addProviderRequirementsFromProviderBlock(reqs getproviders.Requirements, provider *Provider) hcl.Diagnostics { var diags hcl.Diagnostics @@ -1067,11 +1054,12 @@ func (c *Config) transformOverriddenResourcesForTest(run *TestRun, file *TestFil } if res.Mode != overrideRes.Mode { + // TODO ephemeral - include also the ephemeral resource and the test_file.go#override_ephemeral blockName, targetMode := blockNameOverrideResource, "data" if overrideRes.Mode == addrs.DataResourceMode { blockName, targetMode = blockNameOverrideData, "resource" } - // It could be a warning, but for the sake of consistent UX let's make it an error + //It could be a warning, but for the sake of consistent UX let's make it an error diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: fmt.Sprintf("Unsupported `%v` target in `%v` block", targetMode, blockName), diff --git a/internal/configs/config_test.go b/internal/configs/config_test.go index 95a45e86b8..47bb34e442 100644 --- a/internal/configs/config_test.go +++ b/internal/configs/config_test.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "testing" @@ -718,6 +719,81 @@ func TestConfigAddProviderRequirements(t *testing.T) { qualifs := new(getproviders.ProvidersQualification) diags = cfg.addProviderRequirements(reqs, qualifs, true, false) assertNoDiagnostics(t, diags) + if got, want := len(qualifs.Explicit), 1; got != want { + t.Fatalf("expected to have %d explicit provider requirement but got %d", want, got) + } + if got, want := len(qualifs.Implicit), 4; got != want { + t.Fatalf("expected to have %d explicit provider requirement but got %d", want, got) + } + + checks := []struct { + key addrs.Provider + want []addrs.Resource + }{ + { + // check registry.opentofu.org/hashicorp/aws + key: addrs.NewProvider("registry.opentofu.org", "hashicorp", "aws"), + want: []addrs.Resource{ + cfg.Path.Resource(addrs.ManagedResourceMode, "aws_instance", "foo").Resource, + cfg.Path.Resource(addrs.DataResourceMode, "aws_s3_object", "baz").Resource, + cfg.Path.Resource(addrs.EphemeralResourceMode, "aws_secret", "bar").Resource, + }, + }, + { + // check registry.opentofu.org/hashicorp/null + key: addrs.NewProvider("registry.opentofu.org", "hashicorp", "null"), + want: []addrs.Resource{ + cfg.Path.Resource(addrs.ManagedResourceMode, "null_resource", "foo").Resource, + }, + }, + { + // check registry.opentofu.org/hashicorp/local + key: addrs.NewProvider("registry.opentofu.org", "hashicorp", "local"), + want: []addrs.Resource{ + cfg.Path.Resource(addrs.ManagedResourceMode, "local_file", "foo").Resource, + }, + }, + { + // check registry.opentofu.org/hashicorp/template + key: addrs.NewProvider("registry.opentofu.org", "hashicorp", "template"), + want: []addrs.Resource{ + cfg.Path.Resource(addrs.ManagedResourceMode, "local_file", "bar").Resource, + }, + }, + } + for _, c := range checks { + t.Run(c.key.String(), func(t *testing.T) { + refs := qualifs.Implicit[c.key] + if got, want := len(refs), len(c.want); got != want { + t.Fatalf("expected to find %d implicit references for provider %q but got %d", want, c.key, got) + } + + var refsAddrs []addrs.Resource + for _, ref := range refs { + refsAddrs = append(refsAddrs, ref.CfgRes.Resource) + } + sort.Slice(refsAddrs, func(i, j int) bool { + return refsAddrs[i].Less(refsAddrs[j]) + }) + sort.Slice(c.want, func(i, j int) bool { + return c.want[i].Less(c.want[j]) + }) + if diff := cmp.Diff(refsAddrs, c.want); diff != "" { + t.Fatalf("expected to find specific resources to implicitly reference the provider %s. diff:\n%s", c.key, diff) + } + }) + } + + wantReqs := getproviders.Requirements{ + addrs.NewProvider("registry.opentofu.org", "hashicorp", "template"): nil, + addrs.NewProvider("registry.opentofu.org", "hashicorp", "local"): nil, + addrs.NewProvider("registry.opentofu.org", "hashicorp", "null"): nil, + addrs.NewProvider("registry.opentofu.org", "hashicorp", "aws"): nil, + addrs.NewProvider("registry.opentofu.org", "hashicorp", "test"): nil, + } + if diff := cmp.Diff(wantReqs, reqs); diff != "" { + t.Fatalf("unexected returned providers qualifications: %s", diff) + } } func TestConfigImportProviderClashesWithModules(t *testing.T) { @@ -1088,5 +1164,68 @@ func TestIsCallFromRemote(t *testing.T) { } }) } +} + +func TestParseEphemeralBlocks(t *testing.T) { + p := NewParser(nil) + f, diags := p.LoadConfigFile("testdata/ephemeral-blocks/main.tf") + // check diags + { + if len(diags) != 6 { // 4 lifecycle unallowed attributes, unallowed connection block and unallowed provisioner block + t.Fatalf("expected 6 diagnostics but got only: %d", len(diags)) + } + containsExpectedKeywords := func(diagContent string) bool { + for _, k := range []string{"ignore_changes", "prevent_destroy", "create_before_destroy", "replace_triggered_by", "connection", "provisioner"} { + if strings.Contains(diagContent, k) { + return true + } + } + return false + } + for _, diag := range diags { + if content := diag.Error(); !containsExpectedKeywords(content) { + t.Fatalf("expected diagnostic to contain at least one of the keywords: %s", content) + } + } + } + { + if len(f.EphemeralResources) != 2 { + t.Fatalf("expected 2 ephemeral resources but got only: %d", len(f.EphemeralResources)) + } + for _, er := range f.EphemeralResources { + switch er.Name { + case "foo": + if er.ForEach == nil { + t.Errorf("expected to have a for_each expression but got nothing") + } + case "bar": + attrs, _ := er.Config.JustAttributes() + if _, ok := attrs["attribute"]; !ok { + t.Errorf("expected to have \"attribute\" but could not find it") + } + if _, ok := attrs["attribute2"]; !ok { + t.Errorf("expected to have \"attribute\" but could not find it") + } + if er.Count == nil { + t.Errorf("expected to have a count expression but got nothing") + } + if er.ProviderConfigRef == nil || er.ProviderConfigRef.Addr().String() != "provider.test.name" { + t.Errorf("expected to have \"provider.test.name\" provider alias configured but instead it was: %+v", er.ProviderConfigRef) + } + if len(er.Preconditions) != 1 { + t.Errorf("expected to have one precondition but got %d", len(er.Preconditions)) + } + if len(er.Postconditions) != 1 { + t.Errorf("expected to have one postcondition but got %d", len(er.Postconditions)) + } + if len(er.DependsOn) != 1 { + t.Errorf("expected to have a depends_on traversal but got %d", len(er.Postconditions)) + } + if er.Managed != nil { + t.Errorf("error in the parsing code. Ephemeral resources are not meant to have a managed object") + } + } + } + } } diff --git a/internal/configs/configschema/implied_type.go b/internal/configs/configschema/implied_type.go index 70167c74f7..65a84d87c4 100644 --- a/internal/configs/configschema/implied_type.go +++ b/internal/configs/configschema/implied_type.go @@ -61,6 +61,24 @@ func (b *Block) ContainsSensitive() bool { return false } +// ContainsMarks is a wrapper around Block.ContainsSensitive which adds +// another check for the ephemeral nature of the block. +// The schema attributes cannot be marked as ephemeral, only the whole block +// can have that mark. +// Therefore, we don't need to check the schema recursively. +// +// NOTE: It's important to make the distinction between "schema attributes" and +// "value attributes". +// A schema attribute cannot have the ephemeral mark, but a value attribute +// can be marked as ephemeral if it's referencing attribute(s) from another +// ephemeral block. +func (b *Block) ContainsMarks() bool { + if b.Ephemeral { + return true + } + return b.ContainsSensitive() +} + // ImpliedType returns the cty.Type that would result from decoding a Block's // ImpliedType and getting the resulting AttributeType. // diff --git a/internal/configs/configschema/marks.go b/internal/configs/configschema/marks.go index 1a466c3dd9..d94f711539 100644 --- a/internal/configs/configschema/marks.go +++ b/internal/configs/configschema/marks.go @@ -27,6 +27,16 @@ func copyAndExtendPath(path cty.Path, nextSteps ...cty.PathStep) cty.Path { func (b *Block) ValueMarks(val cty.Value, path cty.Path) []cty.PathValueMarks { var pvm []cty.PathValueMarks + // When the block is marked as ephemeral, the whole value needs to be marked accordingly. + // Inner attributes should carry no ephemeral mark. + // The ephemerality of the attributes is given by the mark on the val and not by individual marks + // as it's the case for the sensitive mark. + if b.Ephemeral { + pvm = append(pvm, cty.PathValueMarks{ + Path: path, // raw received path is indicating that the whole value needs to be marked as ephemeral. + Marks: cty.NewValueMarks(marks.Ephemeral), + }) + } // We can mark attributes as sensitive even if the value is null for name, attrS := range b.Attributes { if attrS.Sensitive { @@ -156,3 +166,33 @@ func (o *Object) ValueMarks(val cty.Value, path cty.Path) []cty.PathValueMarks { } return pvm } + +// RemoveEphemeralFromWriteOnly gets the value and for the attributes that are +// configured as write-only removes the marks.Ephemeral mark. +// Write-only arguments are only available in managed resources. +// Write-only arguments are the only managed resource's attribute type +// that can reference ephemeral values. +// Also, the provider framework sdk is responsible with nullify these attributes +// before returning back to OpenTofu. +// +// Therefore, before writing the changes/state of a managed resource to its store, +// we want to be sure that the nil value of the attribute is not marked as ephemeral +// in case it got its value from evaluating an expression where an ephemeral value has +// been involved. +func (b *Block) RemoveEphemeralFromWriteOnly(v cty.Value) cty.Value { + unmarkedV, valMarks := v.UnmarkDeepWithPaths() + for _, pathMark := range valMarks { + if _, ok := pathMark.Marks[marks.Ephemeral]; !ok { + continue + } + attr := b.AttributeByPath(pathMark.Path) + if attr == nil { + continue + } + if !attr.WriteOnly { + continue + } + delete(pathMark.Marks, marks.Ephemeral) + } + return unmarkedV.MarkWithPaths(valMarks) +} diff --git a/internal/configs/configschema/path_test.go b/internal/configs/configschema/path_test.go index 4793373662..3fa4325a7d 100644 --- a/internal/configs/configschema/path_test.go +++ b/internal/configs/configschema/path_test.go @@ -230,5 +230,4 @@ func TestObject_AttributeByPath(t *testing.T) { } }) } - } diff --git a/internal/configs/configschema/schema.go b/internal/configs/configschema/schema.go index 968e20e6bd..a79ffca5f4 100644 --- a/internal/configs/configschema/schema.go +++ b/internal/configs/configschema/schema.go @@ -38,6 +38,11 @@ type Block struct { DescriptionKind StringKind Deprecated bool + + // Ephemeral is a flag indicating that this is an ephemeral block marking it as an "ephemeral context". + // There are multiple places where this is set to "true". Generally speaking, + // any Block that is meant to accept ephemeral values should have this set as "true". + Ephemeral bool } // Attribute represents a configuration attribute, within a block. @@ -94,6 +99,8 @@ type Attribute struct { Sensitive bool Deprecated bool + + WriteOnly bool } // Object represents the embedding of a structural object inside an Attribute. diff --git a/internal/configs/module.go b/internal/configs/module.go index 8997541acf..66f96b645b 100644 --- a/internal/configs/module.go +++ b/internal/configs/module.go @@ -50,8 +50,9 @@ type Module struct { ModuleCalls map[string]*ModuleCall - ManagedResources map[string]*Resource - DataResources map[string]*Resource + ManagedResources map[string]*Resource + DataResources map[string]*Resource + EphemeralResources map[string]*Resource Moved []*Moved Import []*Import @@ -105,8 +106,9 @@ type File struct { ModuleCalls []*ModuleCall - ManagedResources []*Resource - DataResources []*Resource + ManagedResources []*Resource + DataResources []*Resource + EphemeralResources []*Resource Moved []*Moved Import []*Import @@ -178,6 +180,7 @@ func NewModule(primaryFiles, overrideFiles []*File, call StaticModuleCall, sourc ModuleCalls: map[string]*ModuleCall{}, ManagedResources: map[string]*Resource{}, DataResources: map[string]*Resource{}, + EphemeralResources: map[string]*Resource{}, Checks: map[string]*Check{}, ProviderMetas: map[addrs.Provider]*ProviderMeta{}, Tests: map[string]*TestFile{}, @@ -273,6 +276,8 @@ func (m *Module) ResourceByAddr(addr addrs.Resource) *Resource { return m.ManagedResources[key] case addrs.DataResourceMode: return m.DataResources[key] + case addrs.EphemeralResourceMode: + return m.EphemeralResources[key] default: return nil } @@ -466,6 +471,35 @@ func (m *Module) appendFile(file *File) hcl.Diagnostics { m.DataResources[key] = r } + for _, r := range file.EphemeralResources { + key := r.moduleUniqueKey() + if existing, exists := m.EphemeralResources[key]; exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Duplicate ephemeral resource %q configuration", existing.Type), + Detail: fmt.Sprintf("A %s ephemeral resource named %q was already declared at %s. Resource names must be unique per type in each module.", existing.Type, existing.Name, existing.DeclRange), + Subject: &r.DeclRange, + }) + continue + } + m.EphemeralResources[key] = r + + // set the provider FQN for the resource + if r.ProviderConfigRef != nil { + r.Provider = m.ProviderForLocalConfig(r.ProviderConfigAddr()) + } else { + // an invalid resource name (for e.g. "null resource" instead of + // "null_resource") can cause a panic down the line in addrs: + // https://github.com/hashicorp/terraform/issues/25560 + implied, err := addrs.ParseProviderPart(r.Addr().ImpliedProvider()) + if err == nil { + r.Provider = m.ImpliedProviderForUnqualifiedType(implied) + } + // We don't return a diagnostic because the invalid resource name + // will already have been caught. + } + } + for _, c := range file.Checks { if c.DataResource != nil { key := c.DataResource.moduleUniqueKey() @@ -736,6 +770,22 @@ func (m *Module) mergeFile(file *File) hcl.Diagnostics { diags = append(diags, mergeDiags...) } + for _, r := range file.EphemeralResources { + key := r.moduleUniqueKey() + existing, exists := m.EphemeralResources[key] + if !exists { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing ephemeral resource to override", + Detail: fmt.Sprintf("There is no %s ephemeral resource named %q. An override file can only override an ephemeral block defined in a primary configuration file.", r.Type, r.Name), + Subject: &r.DeclRange, + }) + continue + } + mergeDiags := existing.merge(r, m.ProviderRequirements.RequiredProviders) + diags = append(diags, mergeDiags...) + } + for _, m := range file.Moved { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, diff --git a/internal/configs/module_merge.go b/internal/configs/module_merge.go index 767378e41c..9aca8e87b2 100644 --- a/internal/configs/module_merge.go +++ b/internal/configs/module_merge.go @@ -54,6 +54,10 @@ func (v *Variable) merge(ov *Variable) hcl.Diagnostics { if ov.Deprecated != "" { v.Deprecated = ov.Deprecated } + if ov.EphemeralSet { + v.EphemeralSet = ov.EphemeralSet + v.Ephemeral = ov.Ephemeral + } if ov.Default != cty.NilVal { v.Default = ov.Default } @@ -156,6 +160,10 @@ func (o *Output) merge(oo *Output) hcl.Diagnostics { if oo.Deprecated != "" { o.Deprecated = oo.Deprecated } + if oo.EphemeralSet { + o.EphemeralSet = oo.EphemeralSet + o.Ephemeral = oo.Ephemeral + } // We don't allow depends_on to be overridden because that is likely to // cause confusing misbehavior. diff --git a/internal/configs/module_merge_test.go b/internal/configs/module_merge_test.go index e4511eba9a..c476f67c53 100644 --- a/internal/configs/module_merge_test.go +++ b/internal/configs/module_merge_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/opentofu/opentofu/internal/addrs" "github.com/zclconf/go-cty/cty" ) @@ -33,6 +34,8 @@ func TestModuleOverrideVariable(t *testing.T) { Deprecated: "b_override deprecated", Nullable: false, NullableSet: true, + Ephemeral: false, + EphemeralSet: true, Type: cty.String, ConstraintType: cty.String, ParsingMode: VariableParseLiteral, @@ -79,6 +82,125 @@ func TestModuleOverrideVariable(t *testing.T) { assertResultDeepEqual(t, got, want) } +func TestModuleOverrideOutput(t *testing.T) { + mod, diags := testModuleFromDir("testdata/valid-modules/override-output") + assertNoDiagnostics(t, diags) + if mod == nil { + t.Fatalf("module is nil") + } + + got := mod.Outputs + want := map[string]*Output{ + "fully_overridden": { + Name: "fully_overridden", + Description: "b_override description", + DescriptionSet: true, + Expr: &hclsyntax.TemplateExpr{ + Parts: []hclsyntax.Expression{ + &hclsyntax.LiteralValueExpr{ + Val: cty.StringVal("b_override"), + SrcRange: hcl.Range{ + Filename: "testdata/valid-modules/override-output/b_override.tf", + Start: hcl.Pos{ + Line: 2, + Column: 12, + Byte: 39, + }, + End: hcl.Pos{ + Line: 2, + Column: 22, + Byte: 49, + }, + }, + }, + }, + SrcRange: hcl.Range{ + Filename: "testdata/valid-modules/override-output/b_override.tf", + Start: hcl.Pos{ + Line: 2, + Column: 11, + Byte: 38, + }, + End: hcl.Pos{ + Line: 2, + Column: 23, + Byte: 50, + }, + }, + }, + Deprecated: "b_override deprecated", + Ephemeral: false, + EphemeralSet: true, + DeclRange: hcl.Range{ + Filename: filepath.FromSlash("testdata/valid-modules/override-output/primary.tf"), + Start: hcl.Pos{ + Line: 1, + Column: 1, + Byte: 0, + }, + End: hcl.Pos{ + Line: 1, + Column: 26, + Byte: 25, + }, + }, + }, + "partially_overridden": { + Name: "partially_overridden", + Description: "base description", + DescriptionSet: true, + Expr: &hclsyntax.TemplateExpr{ + Parts: []hclsyntax.Expression{ + &hclsyntax.LiteralValueExpr{ + Val: cty.StringVal("b_override partial"), + SrcRange: hcl.Range{ + Filename: "testdata/valid-modules/override-output/b_override.tf", + Start: hcl.Pos{ + Line: 9, + Column: 12, + Byte: 197, + }, + End: hcl.Pos{ + Line: 9, + Column: 30, + Byte: 215, + }, + }, + }, + }, + SrcRange: hcl.Range{ + Filename: "testdata/valid-modules/override-output/b_override.tf", + Start: hcl.Pos{ + Line: 9, + Column: 11, + Byte: 196, + }, + End: hcl.Pos{ + Line: 9, + Column: 31, + Byte: 216, + }, + }, + }, + Deprecated: "b_override deprecated", + DeclRange: hcl.Range{ + Filename: filepath.FromSlash("testdata/valid-modules/override-output/primary.tf"), + Start: hcl.Pos{ + Line: 6, + Column: 1, + Byte: 83, + }, + End: hcl.Pos{ + Line: 6, + Column: 30, + Byte: 112, + }, + }, + }, + } + assertResultDeepEqual(t, got, want) +} + func TestModuleOverrideModule(t *testing.T) { mod, diags := testModuleFromDir("testdata/valid-modules/override-module") assertNoDiagnostics(t, diags) diff --git a/internal/configs/module_test.go b/internal/configs/module_test.go index 16f14a020f..23c6c3eb73 100644 --- a/internal/configs/module_test.go +++ b/internal/configs/module_test.go @@ -69,6 +69,15 @@ func TestNewModule_resource_providers(t *testing.T) { wantBar := addrs.NewProvider(addrs.DefaultProviderRegistryHost, "bar", "test") // root module + if got, want := len(cfg.Module.ManagedResources), 2; got != want { + t.Fatalf("expected to have %d managed resources in the root module but got %d", want, got) + } + if got, want := len(cfg.Module.DataResources), 1; got != want { + t.Fatalf("expected to have %d data sources in the root module but got %d", want, got) + } + if got, want := len(cfg.Module.EphemeralResources), 2; got != want { + t.Fatalf("expected to have %d ephemeral resources in the root module but got %d", want, got) + } if !cfg.Module.ManagedResources["test_instance.explicit"].Provider.Equals(wantFoo) { t.Fatalf("wrong provider for \"test_instance.explicit\"\ngot: %s\nwant: %s", cfg.Module.ManagedResources["test_instance.explicit"].Provider, @@ -90,8 +99,31 @@ func TestNewModule_resource_providers(t *testing.T) { ) } + // ephemeral resources test + if !cfg.Module.EphemeralResources["ephemeral.test_ephemeral.explicit"].Provider.Equals(wantFoo) { + t.Fatalf("wrong provider for \"test_ephemeral.explicit\"\ngot: %s\nwant: %s", + cfg.Module.EphemeralResources["test_ephemeral.explicit"].Provider, + wantFoo, + ) + } + if !cfg.Module.EphemeralResources["ephemeral.test_ephemeral.implicit"].Provider.Equals(wantImplicit) { + t.Fatalf("wrong provider for \"test_ephemeral.implicit\"\ngot: %s\nwant: %s", + cfg.Module.EphemeralResources["test_instance.implicit"].Provider, + wantImplicit, + ) + } + // child module cm := cfg.Children["child"].Module + if got, want := len(cm.ManagedResources), 3; got != want { + t.Fatalf("expected to have %d managed resources in the child module but got %d", want, got) + } + if got, want := len(cm.DataResources), 0; got != want { + t.Fatalf("expected to have %d data sources in the child module but got %d", want, got) + } + if got, want := len(cm.EphemeralResources), 2; got != want { + t.Fatalf("expected to have %d ephemeral resources in the child module but got %d", want, got) + } if !cm.ManagedResources["test_instance.explicit"].Provider.Equals(wantBar) { t.Fatalf("wrong provider for \"module.child.test_instance.explicit\"\ngot: %s\nwant: %s", cfg.Module.ManagedResources["test_instance.explicit"].Provider, @@ -104,6 +136,19 @@ func TestNewModule_resource_providers(t *testing.T) { wantImplicit, ) } + // ephemeral + if !cm.EphemeralResources["ephemeral.test_ephemeral.other_explicit"].Provider.Equals(wantFoo) { + t.Fatalf("wrong provider for \"module.child.ephemeral.test_ephemeral.other_explicit\"\ngot: %s\nwant: %s", + cfg.Module.EphemeralResources["ephemeral.test_ephemeral.other_explicit"].Provider, + wantFoo, + ) + } + if !cm.EphemeralResources["ephemeral.test_ephemeral.other_implicit"].Provider.Equals(wantImplicit) { + t.Fatalf("wrong provider for \"module.child.ephemeral.test_ephemeral.other_implicit\"\ngot: %s\nwant: %s", + cfg.Module.EphemeralResources["ephemeral.test_ephemeral.other_implicit"].Provider, + wantFoo, + ) + } } func TestProviderForLocalConfig(t *testing.T) { @@ -291,15 +336,21 @@ func TestModule_implied_provider(t *testing.T) { }{ {"foo_resource.a", foo}, {"data.foo_resource.b", foo}, - {"bar_resource.c", bar}, - {"data.bar_resource.d", bar}, - {"whatever_resource.e", whatever}, - {"data.whatever_resource.f", whatever}, + {"ephemeral.foo_resource.c", foo}, + {"bar_resource.d", bar}, + {"data.bar_resource.e", bar}, + {"ephemeral.bar_resource.f", bar}, + {"whatever_resource.g", whatever}, + {"data.whatever_resource.h", whatever}, + {"ephemeral.whatever_resource.i", whatever}, } for _, test := range tests { resources := mod.ManagedResources - if strings.HasPrefix(test.Address, "data.") { + switch test.Address[:strings.Index(test.Address, ".")+1] { + case "data.": resources = mod.DataResources + case "ephemeral.": + resources = mod.EphemeralResources } resource, exists := resources[test.Address] if !exists { @@ -443,3 +494,30 @@ func TestModule_cloud_duplicate_overrides(t *testing.T) { t.Fatalf("expected module error to contain %q\nerror was:\n%s", want, got) } } + +func TestResourceByAddr(t *testing.T) { + managedResource := &Resource{Mode: addrs.ManagedResourceMode, Name: "name", Type: "test_resource"} + dataResource := &Resource{Mode: addrs.DataResourceMode, Name: "name", Type: "test_data"} + ephemeralResource := &Resource{Mode: addrs.EphemeralResourceMode, Name: "name", Type: "test_ephemeral"} + m := Module{ + ManagedResources: map[string]*Resource{ + managedResource.Addr().String(): managedResource, + }, + DataResources: map[string]*Resource{ + dataResource.Addr().String(): dataResource, + }, + EphemeralResources: map[string]*Resource{ + ephemeralResource.Addr().String(): ephemeralResource, + }, + } + if got, want := m.ResourceByAddr(managedResource.Addr()), managedResource; got != want { + t.Fatalf("expected resource %+v but got %+v", want, got) + } + if got, want := m.ResourceByAddr(dataResource.Addr()), dataResource; got != want { + t.Fatalf("expected resource %+v but got %+v", want, got) + } + if got, want := m.ResourceByAddr(ephemeralResource.Addr()), ephemeralResource; got != want { + t.Fatalf("expected resource %+v but got %+v", want, got) + } + +} diff --git a/internal/configs/moved.go b/internal/configs/moved.go index d6dca11a97..21e212278a 100644 --- a/internal/configs/moved.go +++ b/internal/configs/moved.go @@ -45,6 +45,23 @@ func decodeMovedBlock(block *hcl.Block) (*Moved, hcl.Diagnostics) { moved.To = to } } + // ensure that the moved block is not used against ephemeral resources since there is no use against those + if !moved.From.SubjectAllowed() { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"moved\" address", + Detail: "The resource referenced by the \"from\" attribute is not allowed to be moved.", + Subject: &moved.DeclRange, + }) + } + if !moved.To.SubjectAllowed() { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid \"moved\" address", + Detail: "The resource referenced by the \"to\" attribute is not allowed to be moved.", + Subject: &moved.DeclRange, + }) + } // we can only move from a module to a module, resource to resource, etc. if !diags.HasErrors() { diff --git a/internal/configs/moved_test.go b/internal/configs/moved_test.go index d4c3fef9d0..dc164f5836 100644 --- a/internal/configs/moved_test.go +++ b/internal/configs/moved_test.go @@ -30,6 +30,8 @@ func TestMovedBlock_decode(t *testing.T) { mod_foo_expr := hcltest.MockExprTraversalSrc("module.foo") mod_bar_expr := hcltest.MockExprTraversalSrc("module.bar") + ephemeral_ref_expr := hcltest.MockExprTraversalSrc("ephemeral.test.test") + tests := map[string]struct { input *hcl.Block want *Moved @@ -150,6 +152,30 @@ func TestMovedBlock_decode(t *testing.T) { }, "Invalid \"moved\" addresses", }, + "error: ephemeral not allowed": { + &hcl.Block{ + Type: "moved", + Body: hcltest.MockBody(&hcl.BodyContent{ + Attributes: hcl.Attributes{ + "to": { + Name: "to", + Expr: ephemeral_ref_expr, + }, + "from": { + Name: "from", + Expr: ephemeral_ref_expr, + }, + }, + }), + DefRange: blockRange, + }, + &Moved{ + To: mustMoveEndpointFromExpr(ephemeral_ref_expr), + From: mustMoveEndpointFromExpr(ephemeral_ref_expr), + DeclRange: blockRange, + }, + "Invalid \"moved\" address", + }, } for name, test := range tests { diff --git a/internal/configs/named_values.go b/internal/configs/named_values.go index 8975c4cce8..61c18a5854 100644 --- a/internal/configs/named_values.go +++ b/internal/configs/named_values.go @@ -40,9 +40,11 @@ type Variable struct { Validations []*CheckRule Sensitive bool Deprecated string + Ephemeral bool DescriptionSet bool SensitiveSet bool + EphemeralSet bool // Nullable indicates that null is a valid value for this variable. Setting // Nullable to false means that the module can expect this variable to @@ -138,6 +140,12 @@ func decodeVariableBlock(block *hcl.Block, override bool) (*Variable, hcl.Diagno } } + if attr, exists := content.Attributes["ephemeral"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &v.Ephemeral) + diags = append(diags, valDiags...) + v.EphemeralSet = true + } + if attr, exists := content.Attributes["nullable"]; exists { valDiags := gohcl.DecodeExpression(attr.Expr, nil, &v.Nullable) diags = append(diags, valDiags...) @@ -418,11 +426,13 @@ type Output struct { DependsOn []hcl.Traversal Sensitive bool Deprecated string + Ephemeral bool Preconditions []*CheckRule DescriptionSet bool SensitiveSet bool + EphemeralSet bool DeclRange hcl.Range @@ -490,6 +500,12 @@ func decodeOutputBlock(block *hcl.Block, override bool) (*Output, hcl.Diagnostic } } + if attr, exists := content.Attributes["ephemeral"]; exists { + valDiags := gohcl.DecodeExpression(attr.Expr, nil, &o.Ephemeral) + diags = append(diags, valDiags...) + o.EphemeralSet = true + } + if attr, exists := content.Attributes["depends_on"]; exists { deps, depsDiags := decodeDependsOn(attr) diags = append(diags, depsDiags...) @@ -523,6 +539,17 @@ func (o *Output) Addr() addrs.OutputValue { return addrs.OutputValue{Name: o.Name} } +// UsageRange returns the location where the output value is configured, but if the expression is not configured +// then it returns the output definition location. +// Useful for generating diagnostics. +func (o *Output) UsageRange() hcl.Range { + subj := o.DeclRange + if o.Expr != nil { + subj = o.Expr.Range() + } + return subj +} + // Local represents a single entry from a "locals" block in a module or file. // The "locals" block itself is not represented, because it serves only to // provide context for us to interpret its contents. @@ -581,6 +608,9 @@ var variableBlockSchema = &hcl.BodySchema{ { Name: "sensitive", }, + { + Name: "ephemeral", + }, { Name: "deprecated", }, @@ -610,6 +640,9 @@ var outputBlockSchema = &hcl.BodySchema{ { Name: "sensitive", }, + { + Name: "ephemeral", + }, { Name: "deprecated", }, diff --git a/internal/configs/parser_config.go b/internal/configs/parser_config.go index 4e4ae35640..e724f1580d 100644 --- a/internal/configs/parser_config.go +++ b/internal/configs/parser_config.go @@ -183,6 +183,13 @@ func (p *Parser) loadConfigFile(path string, override bool) (*File, hcl.Diagnost file.DataResources = append(file.DataResources, cfg) } + case "ephemeral": + cfg, cfgDiags := decodeEphemeralBlock(block, override) + diags = append(diags, cfgDiags...) + if cfg != nil { + file.EphemeralResources = append(file.EphemeralResources, cfg) + } + case "moved": cfg, cfgDiags := decodeMovedBlock(block) diags = append(diags, cfgDiags...) @@ -298,6 +305,10 @@ var configFileSchema = &hcl.BodySchema{ Type: "data", LabelNames: []string{"type", "name"}, }, + { + Type: "ephemeral", + LabelNames: []string{"type", "name"}, + }, { Type: "moved", }, diff --git a/internal/configs/provider_validation.go b/internal/configs/provider_validation.go index 6fe80c3e53..6ad23605ed 100644 --- a/internal/configs/provider_validation.go +++ b/internal/configs/provider_validation.go @@ -458,6 +458,7 @@ func validateProviderConfigs(parentCall *ModuleCall, cfg *Config, noProviderConf } checkImpliedProviderNames(mod.ManagedResources) checkImpliedProviderNames(mod.DataResources) + checkImpliedProviderNames(mod.EphemeralResources) // collect providers passed from the parent if parentCall != nil { @@ -523,6 +524,7 @@ func validateProviderConfigs(parentCall *ModuleCall, cfg *Config, noProviderConf } checkProviderKeys(mod.ManagedResources) checkProviderKeys(mod.DataResources) + checkProviderKeys(mod.EphemeralResources) // Verify that any module calls only refer to named providers, and that // those providers will have a configuration at runtime. This way we can diff --git a/internal/configs/resource.go b/internal/configs/resource.go index cf0e08ae86..37a773e52b 100644 --- a/internal/configs/resource.go +++ b/internal/configs/resource.go @@ -547,6 +547,176 @@ func decodeDataBlock(block *hcl.Block, override, nested bool) (*Resource, hcl.Di return r, diags } +func decodeEphemeralBlock(block *hcl.Block, override bool) (*Resource, hcl.Diagnostics) { + var diags hcl.Diagnostics + r := &Resource{ + Mode: addrs.EphemeralResourceMode, + Type: block.Labels[0], + Name: block.Labels[1], + DeclRange: block.DefRange, + TypeRange: block.LabelRanges[0], + } + + content, remain, moreDiags := block.Body.PartialContent(ResourceBlockSchema) + diags = append(diags, moreDiags...) + r.Config = remain + + if !hclsyntax.ValidIdentifier(r.Type) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid ephemeral resource type name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[0], + }) + } + if !hclsyntax.ValidIdentifier(r.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid ephemeral resource name", + Detail: badIdentifierDetail, + Subject: &block.LabelRanges[1], + }) + } + + if attr, exists := content.Attributes["count"]; exists { + r.Count = attr.Expr + } + + if attr, exists := content.Attributes["for_each"]; exists { + r.ForEach = attr.Expr + // Cannot have count and for_each on the same resource block + if r.Count != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: `Invalid combination of "count" and "for_each"`, + Detail: `The "count" and "for_each" meta-arguments are mutually-exclusive, only one should be used to be explicit about the number of resources to be created.`, + Subject: &attr.NameRange, + }) + } + } + + if attr, exists := content.Attributes["provider"]; exists { + var providerDiags hcl.Diagnostics + r.ProviderConfigRef, providerDiags = decodeProviderConfigRef(attr.Expr, "provider") + diags = append(diags, providerDiags...) + } + + if attr, exists := content.Attributes["depends_on"]; exists { + deps, depsDiags := decodeDependsOn(attr) + diags = append(diags, depsDiags...) + r.DependsOn = append(r.DependsOn, deps...) + } + + invalidEphemeralLifecycleAttributeDiag := func(field string) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid lifecycle configuration for ephemeral resource", + Detail: fmt.Sprintf("The lifecycle argument %q cannot be used in ephemeral resources. This is meant to be used strictly in \"resource\" blocks.", field), + Subject: &block.DefRange, + } + } + invalidEphemeralBlockDiag := func(field string) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid configuration block for ephemeral resource", + Detail: fmt.Sprintf("The block type %q cannot be used in ephemeral resources. This is meant to be used strictly in \"resource\" blocks.", field), + Subject: &block.DefRange, + } + } + var seenLifecycle *hcl.Block + var seenEscapeBlock *hcl.Block + for _, block := range content.Blocks { + switch block.Type { + case "lifecycle": + if seenLifecycle != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate lifecycle block", + Detail: fmt.Sprintf("This ephemeral resource already has a lifecycle block at %s.", seenLifecycle.DefRange), + Subject: &block.DefRange, + }) + continue + } + seenLifecycle = block + + lcContent, lcDiags := block.Body.Content(resourceLifecycleBlockSchema) + diags = append(diags, lcDiags...) + + if _, exists := lcContent.Attributes["create_before_destroy"]; exists { + diags = append(diags, invalidEphemeralLifecycleAttributeDiag("create_before_destroy")) + } + if _, exists := lcContent.Attributes["prevent_destroy"]; exists { + diags = append(diags, invalidEphemeralLifecycleAttributeDiag("prevent_destroy")) + } + if _, exists := lcContent.Attributes["replace_triggered_by"]; exists { + diags = append(diags, invalidEphemeralLifecycleAttributeDiag("replace_triggered_by")) + } + if _, exists := lcContent.Attributes["ignore_changes"]; exists { + diags = append(diags, invalidEphemeralLifecycleAttributeDiag("ignore_changes")) + } + for _, block := range lcContent.Blocks { + switch block.Type { + case "precondition", "postcondition": + cr, moreDiags := decodeCheckRuleBlock(block, override) + diags = append(diags, moreDiags...) + + moreDiags = cr.validateSelfReferences(block.Type, r.Addr()) + diags = append(diags, moreDiags...) + + switch block.Type { + case "precondition": + r.Preconditions = append(r.Preconditions, cr) + case "postcondition": + r.Postconditions = append(r.Postconditions, cr) + } + default: + // The cases above should be exhaustive for all block types + // defined in the lifecycle schema, so this shouldn't happen. + panic(fmt.Sprintf("unexpected lifecycle sub-block type %q", block.Type)) + } + } + + case "connection": + diags = append(diags, invalidEphemeralBlockDiag("connection")) + + case "provisioner": + diags = append(diags, invalidEphemeralBlockDiag("provisioner")) + + case "_": + if seenEscapeBlock != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate escaping block", + Detail: fmt.Sprintf( + "The special block type \"_\" can be used to force particular arguments to be interpreted as resource-type-specific rather than as meta-arguments, but each resource block can have only one such block. The first escaping block was at %s.", + seenEscapeBlock.DefRange, + ), + Subject: &block.DefRange, + }) + continue + } + seenEscapeBlock = block + + // When there's an escaping block its content merges with the + // existing config we extracted earlier, so later decoding + // will see a blend of both. + r.Config = hcl.MergeBodies([]hcl.Body{r.Config, block.Body}) + + default: + // Any other block types are ones we've reserved for future use, + // so they get a generic message. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Reserved block type name in ephemeral block", + Detail: fmt.Sprintf("The block type name %q is reserved for use by OpenTofu in a future version.", block.Type), + Subject: &block.TypeRange, + }) + } + } + + return r, diags +} + // decodeReplaceTriggeredBy decodes and does basic validation of the // replace_triggered_by expressions, ensuring they only contains references to // a single resource, and the only extra variables are count.index or each.key. diff --git a/internal/configs/test_file.go b/internal/configs/test_file.go index 1f813d7894..90186b45ca 100644 --- a/internal/configs/test_file.go +++ b/internal/configs/test_file.go @@ -260,6 +260,7 @@ type TestRunOptions struct { const ( blockNameOverrideResource = "override_resource" blockNameOverrideData = "override_data" + //blockNameOverrideEphemeral = "override_ephemeral" // TODO ephemeral uncomment this when testing support will be added for ephemerals ) // OverrideResource contains information about a resource or data block to be overridden. diff --git a/internal/configs/testdata/ephemeral-blocks/main.tf b/internal/configs/testdata/ephemeral-blocks/main.tf new file mode 100644 index 0000000000..e3332730dd --- /dev/null +++ b/internal/configs/testdata/ephemeral-blocks/main.tf @@ -0,0 +1,40 @@ +provider test { + alias = "name" +} + +ephemeral "test_resource" "foo" { + for_each = toset(["a"]) +} + +ephemeral "test_resource" "bar" { + depends_on = [ + test_resource.foo["a"] + ] + provider = test.name + count = 1 + attribute = "test value" + attribute2 = "test value" + + connection { + // connection blocks are not allowed so we are expecting an error on this + } + provisioner "local-exec" { + // provisioner blocks are not allowed so we are expecting an error on this + } + lifecycle { + // standard attributes in the lifecycle block are not allowed so we are expecting 4 errors on this + create_before_destroy = true + prevent_destroy = true + replace_triggered_by = true + ignore_changes = true + // precondition and postconditions are allowed in ephemeral resources + precondition { + condition = ephemeral.test_resource.foo.id == "" + error_message = "precondition error" + } + postcondition { + condition = ephemeral.test_resource.foo.id == "" + error_message = "postcondition error" + } + } +} \ No newline at end of file diff --git a/internal/configs/testdata/valid-files/providers-explicit-implied.tf b/internal/configs/testdata/valid-files/providers-explicit-implied.tf index 49c063a1e2..6611e5287d 100644 --- a/internal/configs/testdata/valid-files/providers-explicit-implied.tf +++ b/internal/configs/testdata/valid-files/providers-explicit-implied.tf @@ -14,6 +14,10 @@ resource "null_resource" "foo" { } +ephemeral "aws_secret" "bar" {} + +data "aws_s3_object" "baz" {} + import { id = "directory/filename" to = local_file.foo diff --git a/internal/configs/testdata/valid-modules/implied-providers/resources.tf b/internal/configs/testdata/valid-modules/implied-providers/resources.tf index 6b4a8af87a..0fe7b077c5 100644 --- a/internal/configs/testdata/valid-modules/implied-providers/resources.tf +++ b/internal/configs/testdata/valid-modules/implied-providers/resources.tf @@ -1,12 +1,15 @@ // These resources map to the configured "foo" provider" resource foo_resource "a" {} data foo_resource "b" {} +ephemeral foo_resource "c" {} // These resources map to a default "hashicorp/bar" provider -resource bar_resource "c" {} -data bar_resource "d" {} +resource bar_resource "d" {} +data bar_resource "e" {} +ephemeral bar_resource "f" {} // These resources map to the configured "whatever" provider, which has FQN // "acme/something". -resource whatever_resource "e" {} -data whatever_resource "f" {} +resource whatever_resource "g" {} +data whatever_resource "h" {} +ephemeral whatever_resource "i" {} diff --git a/internal/configs/testdata/valid-modules/nested-providers-fqns/child/main.tf b/internal/configs/testdata/valid-modules/nested-providers-fqns/child/main.tf index 79e449bf46..44a3cf8b38 100644 --- a/internal/configs/testdata/valid-modules/nested-providers-fqns/child/main.tf +++ b/internal/configs/testdata/valid-modules/nested-providers-fqns/child/main.tf @@ -23,3 +23,12 @@ resource "test_instance" "implicit" { resource "test_instance" "other" { provider = foo-test.other } + +ephemeral "test_ephemeral" "other_explicit" { + provider = foo-test.other +} + +ephemeral "test_ephemeral" "other_implicit" { + // since the provider type name "test" does not match an entry in + // required_providers, the default provider "test" should be used +} diff --git a/internal/configs/testdata/valid-modules/nested-providers-fqns/main.tf b/internal/configs/testdata/valid-modules/nested-providers-fqns/main.tf index 27988f42b2..bcbadc0a72 100644 --- a/internal/configs/testdata/valid-modules/nested-providers-fqns/main.tf +++ b/internal/configs/testdata/valid-modules/nested-providers-fqns/main.tf @@ -23,6 +23,15 @@ data "test_resource" "explicit" { provider = foo-test } +ephemeral "test_ephemeral" "explicit" { + provider = foo-test +} + +ephemeral "test_ephemeral" "implicit" { + // since the provider type name "test" does not match an entry in + // required_providers, the default provider "test" should be used +} + resource "test_instance" "implicit" { // since the provider type name "test" does not match an entry in // required_providers, the default provider "test" should be used diff --git a/internal/configs/testdata/valid-modules/override-output/a_override.tf b/internal/configs/testdata/valid-modules/override-output/a_override.tf new file mode 100644 index 0000000000..489eda5258 --- /dev/null +++ b/internal/configs/testdata/valid-modules/override-output/a_override.tf @@ -0,0 +1,10 @@ +output "fully_overridden" { + value = "a_override" + description = "a_override description" + deprecated = "a_override deprecated" + ephemeral = true +} + +output "partially_overridden" { + value = "a_override partial" +} diff --git a/internal/configs/testdata/valid-modules/override-output/b_override.tf b/internal/configs/testdata/valid-modules/override-output/b_override.tf new file mode 100644 index 0000000000..e066c0a99f --- /dev/null +++ b/internal/configs/testdata/valid-modules/override-output/b_override.tf @@ -0,0 +1,11 @@ +output "fully_overridden" { + value = "b_override" + description = "b_override description" + deprecated = "b_override deprecated" + ephemeral = false +} + +output "partially_overridden" { + value = "b_override partial" + deprecated = "b_override deprecated" +} diff --git a/internal/configs/testdata/valid-modules/override-output/primary.tf b/internal/configs/testdata/valid-modules/override-output/primary.tf new file mode 100644 index 0000000000..b9a6d75249 --- /dev/null +++ b/internal/configs/testdata/valid-modules/override-output/primary.tf @@ -0,0 +1,9 @@ +output "fully_overridden" { + value = "base" + description = "base description" +} + +output "partially_overridden" { + value = "base" + description = "base description" +} diff --git a/internal/configs/testdata/valid-modules/override-variable/a_override.tf b/internal/configs/testdata/valid-modules/override-variable/a_override.tf index 7632bd2419..aa3085659d 100644 --- a/internal/configs/testdata/valid-modules/override-variable/a_override.tf +++ b/internal/configs/testdata/valid-modules/override-variable/a_override.tf @@ -3,6 +3,7 @@ variable "fully_overridden" { description = "a_override description" deprecated = "a_override deprecated" type = string + ephemeral = true } variable "partially_overridden" { diff --git a/internal/configs/testdata/valid-modules/override-variable/b_override.tf b/internal/configs/testdata/valid-modules/override-variable/b_override.tf index 5cf7bb98e9..c1225a5d15 100644 --- a/internal/configs/testdata/valid-modules/override-variable/b_override.tf +++ b/internal/configs/testdata/valid-modules/override-variable/b_override.tf @@ -4,6 +4,7 @@ variable "fully_overridden" { description = "b_override description" deprecated = "b_override deprecated" type = string + ephemeral = false } variable "partially_overridden" { diff --git a/internal/grpcwrap/provider.go b/internal/grpcwrap/provider.go index 08a0a1d4cc..c2063b599f 100644 --- a/internal/grpcwrap/provider.go +++ b/internal/grpcwrap/provider.go @@ -14,6 +14,7 @@ import ( "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" "github.com/zclconf/go-cty/cty/msgpack" + "google.golang.org/protobuf/types/known/timestamppb" ) // New wraps a providers.Interface to implement a grpc ProviderServer. @@ -33,8 +34,9 @@ type provider struct { func (p *provider) GetSchema(_ context.Context, req *tfplugin5.GetProviderSchema_Request) (*tfplugin5.GetProviderSchema_Response, error) { resp := &tfplugin5.GetProviderSchema_Response{ - ResourceSchemas: make(map[string]*tfplugin5.Schema), - DataSourceSchemas: make(map[string]*tfplugin5.Schema), + ResourceSchemas: make(map[string]*tfplugin5.Schema), + DataSourceSchemas: make(map[string]*tfplugin5.Schema), + EphemeralResourceSchemas: make(map[string]*tfplugin5.Schema), } resp.Provider = &tfplugin5.Schema{ @@ -63,6 +65,12 @@ func (p *provider) GetSchema(_ context.Context, req *tfplugin5.GetProviderSchema Block: convert.ConfigSchemaToProto(dat.Block), } } + for typ, dat := range p.schema.EphemeralResources { + resp.EphemeralResourceSchemas[typ] = &tfplugin5.Schema{ + Version: dat.Version, + Block: convert.ConfigSchemaToProto(dat.Block), + } + } resp.ServerCapabilities = &tfplugin5.ServerCapabilities{ PlanDestroy: p.schema.ServerCapabilities.PlanDestroy, @@ -131,6 +139,26 @@ func (p *provider) ValidateDataSourceConfig(ctx context.Context, req *tfplugin5. return resp, nil } +// ValidateEphemeralResourceConfig implements tfplugin5.ProviderServer. +func (p *provider) ValidateEphemeralResourceConfig(ctx context.Context, req *tfplugin5.ValidateEphemeralResourceConfig_Request) (*tfplugin5.ValidateEphemeralResourceConfig_Response, error) { + resp := &tfplugin5.ValidateEphemeralResourceConfig_Response{} + ty := p.schema.EphemeralResources[req.TypeName].Block.ImpliedType() + + configVal, err := decodeDynamicValue(req.Config, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + validateResp := p.provider.ValidateEphemeralConfig(ctx, providers.ValidateEphemeralConfigRequest{ + TypeName: req.TypeName, + Config: configVal, + }) + + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, validateResp.Diagnostics) + return resp, nil +} + func (p *provider) UpgradeResourceState(ctx context.Context, req *tfplugin5.UpgradeResourceState_Request) (*tfplugin5.UpgradeResourceState_Response, error) { resp := &tfplugin5.UpgradeResourceState_Response{} ty := p.schema.ResourceTypes[req.TypeName].Block.ImpliedType() @@ -392,24 +420,69 @@ func (p *provider) ReadDataSource(ctx context.Context, req *tfplugin5.ReadDataSo return resp, nil } -// CloseEphemeralResource implements tfplugin5.ProviderServer. -func (p *provider) CloseEphemeralResource(context.Context, *tfplugin5.CloseEphemeralResource_Request) (*tfplugin5.CloseEphemeralResource_Response, error) { - panic("unimplemented") -} - // OpenEphemeralResource implements tfplugin5.ProviderServer. -func (p *provider) OpenEphemeralResource(context.Context, *tfplugin5.OpenEphemeralResource_Request) (*tfplugin5.OpenEphemeralResource_Response, error) { - panic("unimplemented") +func (p *provider) OpenEphemeralResource(ctx context.Context, req *tfplugin5.OpenEphemeralResource_Request) (*tfplugin5.OpenEphemeralResource_Response, error) { + resp := &tfplugin5.OpenEphemeralResource_Response{} + ty := p.schema.EphemeralResources[req.TypeName].Block.ImpliedType() + + configVal, err := decodeDynamicValue(req.Config, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + openResp := p.provider.OpenEphemeralResource(ctx, providers.OpenEphemeralResourceRequest{ + TypeName: req.TypeName, + Config: configVal, + }) + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, openResp.Diagnostics) + if openResp.Diagnostics.HasErrors() { + return resp, nil + } + + resp.Result, err = encodeDynamicValue(openResp.Result, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + resp.Private = openResp.Private + if openResp.RenewAt != nil { + resp.RenewAt = timestamppb.New(*openResp.RenewAt) + } + return resp, nil } // RenewEphemeralResource implements tfplugin5.ProviderServer. -func (p *provider) RenewEphemeralResource(context.Context, *tfplugin5.RenewEphemeralResource_Request) (*tfplugin5.RenewEphemeralResource_Response, error) { - panic("unimplemented") +func (p *provider) RenewEphemeralResource(ctx context.Context, req *tfplugin5.RenewEphemeralResource_Request) (*tfplugin5.RenewEphemeralResource_Response, error) { + resp := &tfplugin5.RenewEphemeralResource_Response{} + + renewResp := p.provider.RenewEphemeralResource(ctx, providers.RenewEphemeralResourceRequest{ + TypeName: req.TypeName, + Private: req.Private, + }) + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, renewResp.Diagnostics) + if renewResp.Diagnostics.HasErrors() { + return resp, nil + } + + resp.Private = renewResp.Private + if renewResp.RenewAt != nil { + resp.RenewAt = timestamppb.New(*renewResp.RenewAt) + } + return resp, nil } -// ValidateEphemeralResourceConfig implements tfplugin5.ProviderServer. -func (p *provider) ValidateEphemeralResourceConfig(context.Context, *tfplugin5.ValidateEphemeralResourceConfig_Request) (*tfplugin5.ValidateEphemeralResourceConfig_Response, error) { - panic("unimplemented") +// CloseEphemeralResource implements tfplugin5.ProviderServer. +func (p *provider) CloseEphemeralResource(ctx context.Context, req *tfplugin5.CloseEphemeralResource_Request) (*tfplugin5.CloseEphemeralResource_Response, error) { + resp := &tfplugin5.CloseEphemeralResource_Response{} + + renewResp := p.provider.CloseEphemeralResource(ctx, providers.CloseEphemeralResourceRequest{ + TypeName: req.TypeName, + Private: req.Private, + }) + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, renewResp.Diagnostics) + return resp, nil } func (p *provider) Stop(ctx context.Context, _ *tfplugin5.Stop_Request) (*tfplugin5.Stop_Response, error) { diff --git a/internal/grpcwrap/provider6.go b/internal/grpcwrap/provider6.go index 26f25a347a..22e659a10c 100644 --- a/internal/grpcwrap/provider6.go +++ b/internal/grpcwrap/provider6.go @@ -14,6 +14,7 @@ import ( "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" "github.com/zclconf/go-cty/cty/msgpack" + "google.golang.org/protobuf/types/known/timestamppb" ) // New wraps a providers.Interface to implement a grpc ProviderServer using @@ -33,8 +34,9 @@ type provider6 struct { func (p *provider6) GetProviderSchema(_ context.Context, req *tfplugin6.GetProviderSchema_Request) (*tfplugin6.GetProviderSchema_Response, error) { resp := &tfplugin6.GetProviderSchema_Response{ - ResourceSchemas: make(map[string]*tfplugin6.Schema), - DataSourceSchemas: make(map[string]*tfplugin6.Schema), + ResourceSchemas: make(map[string]*tfplugin6.Schema), + DataSourceSchemas: make(map[string]*tfplugin6.Schema), + EphemeralResourceSchemas: make(map[string]*tfplugin6.Schema), } resp.Provider = &tfplugin6.Schema{ @@ -63,6 +65,12 @@ func (p *provider6) GetProviderSchema(_ context.Context, req *tfplugin6.GetProvi Block: convert.ConfigSchemaToProto(dat.Block), } } + for typ, dat := range p.schema.EphemeralResources { + resp.EphemeralResourceSchemas[typ] = &tfplugin6.Schema{ + Version: dat.Version, + Block: convert.ConfigSchemaToProto(dat.Block), + } + } resp.ServerCapabilities = &tfplugin6.ServerCapabilities{ PlanDestroy: p.schema.ServerCapabilities.PlanDestroy, @@ -131,6 +139,26 @@ func (p *provider6) ValidateDataResourceConfig(ctx context.Context, req *tfplugi return resp, nil } +// ValidateEphemeralResourceConfig implements tfplugin6.ProviderServer. +func (p *provider6) ValidateEphemeralResourceConfig(ctx context.Context, req *tfplugin6.ValidateEphemeralResourceConfig_Request) (*tfplugin6.ValidateEphemeralResourceConfig_Response, error) { + resp := &tfplugin6.ValidateEphemeralResourceConfig_Response{} + ty := p.schema.EphemeralResources[req.TypeName].Block.ImpliedType() + + configVal, err := decodeDynamicValue6(req.Config, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + validateResp := p.provider.ValidateEphemeralConfig(ctx, providers.ValidateEphemeralConfigRequest{ + TypeName: req.TypeName, + Config: configVal, + }) + + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, validateResp.Diagnostics) + return resp, nil +} + func (p *provider6) UpgradeResourceState(ctx context.Context, req *tfplugin6.UpgradeResourceState_Request) (*tfplugin6.UpgradeResourceState_Response, error) { resp := &tfplugin6.UpgradeResourceState_Response{} ty := p.schema.ResourceTypes[req.TypeName].Block.ImpliedType() @@ -392,24 +420,69 @@ func (p *provider6) ReadDataSource(ctx context.Context, req *tfplugin6.ReadDataS return resp, nil } -// CloseEphemeralResource implements tfplugin6.ProviderServer. -func (p *provider6) CloseEphemeralResource(context.Context, *tfplugin6.CloseEphemeralResource_Request) (*tfplugin6.CloseEphemeralResource_Response, error) { - panic("unimplemented") -} - // OpenEphemeralResource implements tfplugin6.ProviderServer. -func (p *provider6) OpenEphemeralResource(context.Context, *tfplugin6.OpenEphemeralResource_Request) (*tfplugin6.OpenEphemeralResource_Response, error) { - panic("unimplemented") +func (p *provider6) OpenEphemeralResource(ctx context.Context, req *tfplugin6.OpenEphemeralResource_Request) (*tfplugin6.OpenEphemeralResource_Response, error) { + resp := &tfplugin6.OpenEphemeralResource_Response{} + ty := p.schema.EphemeralResources[req.TypeName].Block.ImpliedType() + + configVal, err := decodeDynamicValue6(req.Config, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + openResp := p.provider.OpenEphemeralResource(ctx, providers.OpenEphemeralResourceRequest{ + TypeName: req.TypeName, + Config: configVal, + }) + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, openResp.Diagnostics) + if openResp.Diagnostics.HasErrors() { + return resp, nil + } + + resp.Result, err = encodeDynamicValue6(openResp.Result, ty) + if err != nil { + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, err) + return resp, nil + } + + resp.Private = openResp.Private + if openResp.RenewAt != nil { + resp.RenewAt = timestamppb.New(*openResp.RenewAt) + } + return resp, nil } // RenewEphemeralResource implements tfplugin6.ProviderServer. -func (p *provider6) RenewEphemeralResource(context.Context, *tfplugin6.RenewEphemeralResource_Request) (*tfplugin6.RenewEphemeralResource_Response, error) { - panic("unimplemented") +func (p *provider6) RenewEphemeralResource(ctx context.Context, req *tfplugin6.RenewEphemeralResource_Request) (*tfplugin6.RenewEphemeralResource_Response, error) { + resp := &tfplugin6.RenewEphemeralResource_Response{} + + renewResp := p.provider.RenewEphemeralResource(ctx, providers.RenewEphemeralResourceRequest{ + TypeName: req.TypeName, + Private: req.Private, + }) + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, renewResp.Diagnostics) + if renewResp.Diagnostics.HasErrors() { + return resp, nil + } + + resp.Private = renewResp.Private + if renewResp.RenewAt != nil { + resp.RenewAt = timestamppb.New(*renewResp.RenewAt) + } + return resp, nil } -// ValidateEphemeralResourceConfig implements tfplugin6.ProviderServer. -func (p *provider6) ValidateEphemeralResourceConfig(context.Context, *tfplugin6.ValidateEphemeralResourceConfig_Request) (*tfplugin6.ValidateEphemeralResourceConfig_Response, error) { - panic("unimplemented") +// CloseEphemeralResource implements tfplugin6.ProviderServer. +func (p *provider6) CloseEphemeralResource(ctx context.Context, req *tfplugin6.CloseEphemeralResource_Request) (*tfplugin6.CloseEphemeralResource_Response, error) { + resp := &tfplugin6.CloseEphemeralResource_Response{} + + renewResp := p.provider.CloseEphemeralResource(ctx, providers.CloseEphemeralResourceRequest{ + TypeName: req.TypeName, + Private: req.Private, + }) + resp.Diagnostics = convert.AppendProtoDiag(resp.Diagnostics, renewResp.Diagnostics) + return resp, nil } func (p *provider6) StopProvider(ctx context.Context, _ *tfplugin6.StopProvider_Request) (*tfplugin6.StopProvider_Response, error) { diff --git a/internal/lang/eval.go b/internal/lang/eval.go index 1220e95a87..cbfcb21565 100644 --- a/internal/lang/eval.go +++ b/internal/lang/eval.go @@ -8,6 +8,7 @@ package lang import ( "context" "fmt" + "log" "maps" "reflect" "strings" @@ -16,16 +17,15 @@ import ( "github.com/hashicorp/hcl/v2/ext/dynblock" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/convert" - "github.com/zclconf/go-cty/cty/function" - "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/configs/configschema" "github.com/opentofu/opentofu/internal/instances" "github.com/opentofu/opentofu/internal/lang/blocktoattr" "github.com/opentofu/opentofu/internal/lang/marks" "github.com/opentofu/opentofu/internal/tfdiags" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" + "github.com/zclconf/go-cty/cty/function" ) // ExpandBlock expands any "dynamic" blocks present in the given body. The @@ -80,6 +80,8 @@ func (s *Scope) EvalBlock(ctx context.Context, body hcl.Body, schema *configsche val, evalDiags := hcldec.Decode(body, spec, hclCtx) diags = diags.Append(enhanceFunctionDiags(evalDiags)) + diags = diags.Append(validEphemeralReferences(schema, val)) + val, depDiags := marks.ExtractDeprecationDiagnosticsWithBody(val, body) diags = diags.Append(depDiags) @@ -398,35 +400,37 @@ func (s *Scope) evalContext(ctx context.Context, parent *hcl.EvalContext, refs [ type evalVarBuilder struct { s *Scope - dataResources map[string]map[string]cty.Value - managedResources map[string]map[string]cty.Value - wholeModules map[string]cty.Value - inputVariables map[string]cty.Value - localValues map[string]cty.Value - outputValues map[string]cty.Value - pathAttrs map[string]cty.Value - terraformAttrs map[string]cty.Value - countAttrs map[string]cty.Value - forEachAttrs map[string]cty.Value - checkBlocks map[string]cty.Value - self cty.Value + dataResources map[string]map[string]cty.Value + managedResources map[string]map[string]cty.Value + ephemeralResources map[string]map[string]cty.Value + wholeModules map[string]cty.Value + inputVariables map[string]cty.Value + localValues map[string]cty.Value + outputValues map[string]cty.Value + pathAttrs map[string]cty.Value + terraformAttrs map[string]cty.Value + countAttrs map[string]cty.Value + forEachAttrs map[string]cty.Value + checkBlocks map[string]cty.Value + self cty.Value } func (s *Scope) newEvalVarBuilder() *evalVarBuilder { return &evalVarBuilder{ s: s, - dataResources: map[string]map[string]cty.Value{}, - managedResources: map[string]map[string]cty.Value{}, - wholeModules: map[string]cty.Value{}, - inputVariables: map[string]cty.Value{}, - localValues: map[string]cty.Value{}, - outputValues: map[string]cty.Value{}, - pathAttrs: map[string]cty.Value{}, - terraformAttrs: map[string]cty.Value{}, - countAttrs: map[string]cty.Value{}, - forEachAttrs: map[string]cty.Value{}, - checkBlocks: map[string]cty.Value{}, + dataResources: map[string]map[string]cty.Value{}, + ephemeralResources: map[string]map[string]cty.Value{}, + managedResources: map[string]map[string]cty.Value{}, + wholeModules: map[string]cty.Value{}, + inputVariables: map[string]cty.Value{}, + localValues: map[string]cty.Value{}, + outputValues: map[string]cty.Value{}, + pathAttrs: map[string]cty.Value{}, + terraformAttrs: map[string]cty.Value{}, + countAttrs: map[string]cty.Value{}, + forEachAttrs: map[string]cty.Value{}, + checkBlocks: map[string]cty.Value{}, } } @@ -549,6 +553,8 @@ func (b *evalVarBuilder) putResourceValue(ctx context.Context, res addrs.Resourc into = b.managedResources case addrs.DataResourceMode: into = b.dataResources + case addrs.EphemeralResourceMode: + into = b.ephemeralResources case addrs.InvalidResourceMode: panic("BUG: got invalid resource mode") default: @@ -577,6 +583,7 @@ func (b *evalVarBuilder) buildAllVariablesInto(vals map[string]cty.Value) { vals["resource"] = cty.ObjectVal(buildResourceObjects(b.managedResources)) vals["data"] = cty.ObjectVal(buildResourceObjects(b.dataResources)) + vals["ephemeral"] = cty.ObjectVal(buildResourceObjects(b.ephemeralResources)) vals["module"] = cty.ObjectVal(b.wholeModules) vals["var"] = cty.ObjectVal(b.inputVariables) vals["local"] = cty.ObjectVal(b.localValues) @@ -619,3 +626,72 @@ func normalizeRefValue(val cty.Value, diags tfdiags.Diagnostics) (cty.Value, tfd } return val, diags } + +// validEphemeralReferences is checking if val is containing ephemeral marks. +// The schema argument is used to figure out if the value is for an ephemeral +// context. If it is, then we don't even validate ephemeral marks. +// If val contains any ephemeral mark, we check if the attribute containing +// an ephemeral value is a write-only one. If not, we generate a diagnostic. +// The diagnostics returned by this method need to go through InConfigBody +// by the caller of the evaluator to append additional context to the diagnostic +// for an enhanced feedback to the user. +// +// A nil schema will handle the value as unable to hold any ephemeral mark. +func validEphemeralReferences(schema *configschema.Block, val cty.Value) (diags tfdiags.Diagnostics) { + // Ephemeral resources can reference values with any mark, so ignore this validation for ephemeral blocks + if schema != nil && schema.Ephemeral { + return diags + } + // This is the function for schema != nil. + // In the case of schema == nil, the function is recreated below. + // + // In cases of DynamicPseudoType attribute in the schema, the attribute that is actually + // referencing an ephemeral value might be missing from the schema. + // Therefore, we search for the first ancestor that exists in the schema. + attrFromSchema := func(path cty.Path) (*configschema.Attribute, cty.Path) { + attrPath := path + attr := schema.AttributeByPath(attrPath) + for attr == nil { + if len(attrPath) == 0 { + log.Printf("[WARN] no valid path found in schema for path \"%#v\"", path) + return nil, path + } + attrPath = attrPath[:len(attrPath)-1] + attr = schema.AttributeByPath(attrPath) + } + return attr, attrPath + } + + // We recreate the attribute search in the schema here purely for being sure + // that the logic below can run even when the schema is nil. + // When there is no schema, there should be no ephemeral value in the block. + if schema == nil { + attrFromSchema = func(path cty.Path) (*configschema.Attribute, cty.Path) { + return nil, path + } + } + + _, valueMarks := val.UnmarkDeepWithPaths() + for _, pathMark := range valueMarks { + _, ok := pathMark.Marks[marks.Ephemeral] + if !ok { + continue + } + + // If the block is not ephemeral, then only its write-only attributes can reference ephemeral values. + // To figure it out, we need to find the attribute by the mark path. + attr, foundPath := attrFromSchema(pathMark.Path) + if attr != nil && attr.WriteOnly { + continue + } + + diags = diags.Append(tfdiags.AttributeValue( + tfdiags.Error, + "Ephemeral value used in non-ephemeral context", + fmt.Sprintf("Attribute %q is referencing an ephemeral value but ephemeral values can be referenced only by other ephemeral attributes or by write-only ones.", tfdiags.FormatCtyPath(foundPath)), + foundPath, + )) + } + + return diags +} diff --git a/internal/lang/eval_test.go b/internal/lang/eval_test.go index f576d7aed3..5e5fffa1d5 100644 --- a/internal/lang/eval_test.go +++ b/internal/lang/eval_test.go @@ -8,12 +8,16 @@ package lang import ( "bytes" "encoding/json" + "fmt" "reflect" "testing" + "github.com/google/go-cmp/cmp" "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/configs/configschema" "github.com/opentofu/opentofu/internal/instances" + "github.com/opentofu/opentofu/internal/lang/marks" + "github.com/opentofu/opentofu/internal/tfdiags" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hcldec" @@ -59,6 +63,9 @@ func TestScopeEvalContext(t *testing.T) { "null_resource.multi[1]": cty.ObjectVal(map[string]cty.Value{ "attr": cty.StringVal("multi1"), }), + "ephemeral.foo_ephemeral.bar": cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("baz"), + }), }, LocalValues: map[string]cty.Value{ "foo": cty.StringVal("bar"), @@ -373,6 +380,18 @@ func TestScopeEvalContext(t *testing.T) { }), }, }, + { + `ephemeral.foo_ephemeral.bar`, + map[string]cty.Value{ + "ephemeral": cty.ObjectVal(map[string]cty.Value{ + "foo_ephemeral": cty.ObjectVal(map[string]cty.Value{ + "bar": cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("baz"), + }), + }), + }), + }, + }, } for _, test := range tests { @@ -1024,3 +1043,199 @@ func Test_enhanceFunctionDiags(t *testing.T) { }) } } + +func TestValidEphemeralReference(t *testing.T) { + schema := &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": { + Type: cty.String, + }, + "secret": { + Type: cty.String, + }, + "secret_wo": { + Type: cty.String, + WriteOnly: true, + }, + }, + BlockTypes: map[string]*configschema.NestedBlock{ + "nested_simple": { + Block: configschema.Block{ + Attributes: map[string]*configschema.Attribute{}, + BlockTypes: map[string]*configschema.NestedBlock{ + "inner_nested_simple": { + Nesting: configschema.NestingSingle, + Block: configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "inner_nested_simple_attr": { + Type: cty.DynamicPseudoType, + WriteOnly: true, + }, + }, + }, + }, + }, + }, + Nesting: configschema.NestingSingle, + }, + "nested_set": { + Block: configschema.Block{ + Attributes: map[string]*configschema.Attribute{}, + BlockTypes: map[string]*configschema.NestedBlock{ + "inner_nested_set": { + Nesting: configschema.NestingSingle, + Block: configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "inner_nested_set_attr": { + Type: cty.DynamicPseudoType, + WriteOnly: true, + }, + }, + }, + }, + }, + }, + Nesting: configschema.NestingSet, + }, + }, + } + tests := map[string]struct { + schema *configschema.Block + val cty.Value + + want tfdiags.Diagnostics + }{ + "nil schema with no ephemeral mark": { + nil, + cty.UnknownVal(cty.String), + nil, + }, + "nil schema with ephemeral mark": { + nil, + cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id value"), + "secret": cty.StringVal("secret value"), + "secret_wo": cty.StringVal("secret value").Mark(marks.Ephemeral), + }), + tfdiags.Diagnostics{}.Append( + tfdiags.AttributeValue( + tfdiags.Error, + "Ephemeral value used in non-ephemeral context", + fmt.Sprintf("Attribute %q is referencing an ephemeral value but ephemeral values can be referenced only by other ephemeral attributes or by write-only ones.", ".secret_wo"), + cty.Path{cty.GetAttrStep{Name: "secret_wo"}}, + ), + ), + }, + "schema is ephemeral": { + &configschema.Block{ + Ephemeral: true, + }, + cty.UnknownVal(cty.String), + nil, + }, + "no checks if the value contains no ephemeral": { + schema, + cty.StringVal("test"), + nil, + }, + "write only argument is referencing ephemeral value": { + schema, + cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id value"), + "secret": cty.StringVal("secret value"), + "secret_wo": cty.StringVal("secret value").Mark(marks.Ephemeral), + }), + nil, + }, + "error when an write-only and a non-write-only contain ephemeral": { + schema, + cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id value"), + "secret": cty.StringVal("secret value").Mark(marks.Ephemeral), + "secret_wo": cty.StringVal("secret value").Mark(marks.Ephemeral), + }), + tfdiags.Diagnostics{}.Append( + tfdiags.AttributeValue( + tfdiags.Error, + "Ephemeral value used in non-ephemeral context", + fmt.Sprintf("Attribute %q is referencing an ephemeral value but ephemeral values can be referenced only by other ephemeral attributes or by write-only ones.", ".secret"), + cty.Path{cty.GetAttrStep{Name: "secret"}}, + ), + ), + }, + "find the right DynamicPseudoType attribute": { + schema, + cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id value"), + "secret_wo": cty.StringVal("secret value").Mark(marks.Ephemeral), + "nested_simple": cty.ObjectVal(map[string]cty.Value{ + "inner_nested_simple": cty.ObjectVal(map[string]cty.Value{ + "inner_nested_simple_attr": cty.ObjectVal(map[string]cty.Value{ + "attribute_not_in_schema": cty.StringVal("test val").Mark(marks.Ephemeral), + }), + }), + }), + }), + nil, + }, + "error when attribute is not in the schema": { + schema, + cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id value"), + "secret_wo": cty.StringVal("secret value").Mark(marks.Ephemeral), + "nested_simple": cty.ObjectVal(map[string]cty.Value{ + "inner_nested_simple": cty.ObjectVal(map[string]cty.Value{ + "block_not_in_schema": cty.ObjectVal(map[string]cty.Value{ + "attribute_not_in_schema": cty.StringVal("test val").Mark(marks.Ephemeral), + }), + }), + }), + }), + tfdiags.Diagnostics{}.Append( + tfdiags.AttributeValue( + tfdiags.Error, + "Ephemeral value used in non-ephemeral context", + fmt.Sprintf( + `Attribute %q is referencing an ephemeral value but ephemeral values can be referenced only by other ephemeral attributes or by write-only ones.`, + ".nested_simple.inner_nested_simple.block_not_in_schema.attribute_not_in_schema", + ), + cty.GetAttrPath("nested_simple").GetAttr("inner_nested_simple").GetAttr("block_not_in_schema").GetAttr("attribute_not_in_schema"), + ), + ), + }, + } + + lookupAttributeDiag := func(forPath cty.Path, in tfdiags.Diagnostics) tfdiags.Diagnostic { + for _, i := range in { + p := tfdiags.GetAttribute(i) + if p.Equals(forPath) { + return i + } + } + return nil + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + diags := validEphemeralReferences(tt.schema, tt.val) + if want, got := len(tt.want), len(diags); want != got { + t.Errorf("wrong number of diags. want: %d; got: %d", want, got) + } + for _, d := range diags { + attributePath := tfdiags.GetAttribute(d) + wantDiag := lookupAttributeDiag(attributePath, tt.want) + if wantDiag == nil { + t.Errorf("got a diagnostic with a path (%s) that is not expected: %s", attributePath, d) + continue + } + gotDesc := d.Description() + wantDesc := wantDiag.Description() + if diff := cmp.Diff(wantDesc, gotDesc); diff != "" { + t.Errorf("%s: unexpected diff in diagnostic description:\n%s", attributePath, diff) + } + if want, got := d.Severity(), wantDiag.Severity(); want != got { + t.Errorf("%s: wrong severity. want %q; got %q", attributePath, want, got) + } + } + }) + } +} diff --git a/internal/lang/marks/marks.go b/internal/lang/marks/marks.go index 5c04243f51..53449eece9 100644 --- a/internal/lang/marks/marks.go +++ b/internal/lang/marks/marks.go @@ -48,6 +48,10 @@ func Contains(val cty.Value, mark valueMark) bool { // OpenTofu. const Sensitive = valueMark("Sensitive") +// Ephemeral indicates that this value is marked as ephemeral in the context of +// OpenTofu. +const Ephemeral = valueMark("Ephemeral") + // TypeType is used to indicate that the value contains a representation of // another value's type. This is part of the implementation of the console-only // `type` function. @@ -226,3 +230,20 @@ func RemoveDeepDeprecated(val cty.Value) cty.Value { val, _ = unmarkDeepWithPathsDeprecated(val) return val } + +// EnsureNoEphemeralMarks checks all the given paths for the Ephemeral mark. +// If there is at least one path marked as such, this method will return +// an error containing the marked paths. +func EnsureNoEphemeralMarks(pvms []cty.PathValueMarks) error { + var res []string + for _, pvm := range pvms { + if _, ok := pvm.Marks[Ephemeral]; ok { + res = append(res, tfdiags.FormatCtyPath(pvm.Path)) + } + } + + if len(res) > 0 { + return fmt.Errorf("ephemeral marks found at the following paths:\n%s", strings.Join(res, "\n")) + } + return nil +} diff --git a/internal/legacy/tofu/resource.go b/internal/legacy/tofu/resource.go index 3a5350c7f7..2864dbec8a 100644 --- a/internal/legacy/tofu/resource.go +++ b/internal/legacy/tofu/resource.go @@ -126,6 +126,9 @@ func NewInstanceInfo(addr addrs.AbsResourceInstance) *InstanceInfo { if addr.Resource.Resource.Mode == addrs.DataResourceMode { id = "data." + id } + if addr.Resource.Resource.Mode == addrs.EphemeralResourceMode { + panic("ephemeral resources are not meant to be processed by this function. Are you sure that this code should be reused?") + } if addr.Resource.Key != addrs.NoKey { switch k := addr.Resource.Key.(type) { case addrs.IntKey: diff --git a/internal/legacy/tofu/resource_address.go b/internal/legacy/tofu/resource_address.go index c8a51df021..ddba161425 100644 --- a/internal/legacy/tofu/resource_address.go +++ b/internal/legacy/tofu/resource_address.go @@ -140,6 +140,10 @@ func (r *ResourceAddress) MatchesResourceConfig(path addrs.Module, rc *configs.R if rc.Mode != addrs.DataResourceMode { return false } + default: + // NOTE: Even though the ephemeral resources are not supported in the legacy form, we want to be sure + // that it is not handled as the other type of resources. + return false } if r.Type != rc.Type || r.Name != rc.Name { return false @@ -300,6 +304,10 @@ func NewLegacyResourceAddress(addr addrs.AbsResource) *ResourceAddress { case addrs.DataResourceMode: ret.Mode = DataResourceMode default: + // This is also covering the unlikely situation when an ephemeral resource will end up in here. + // This is not meant to happen. However, since this method is not used anymore, we want it to panic + // in case somebody starts using it again in the future. This is to indicate that this is legacy code and that + // is not meant to work with new features without putting additional work into it, if ever needed. panic(fmt.Errorf("cannot shim %s to legacy ResourceMode value", addr.Resource.Mode)) } @@ -338,6 +346,10 @@ func NewLegacyResourceInstanceAddress(addr addrs.AbsResourceInstance) *ResourceA case addrs.DataResourceMode: ret.Mode = DataResourceMode default: + // This is also covering the unlikely situation when an ephemeral resource will end up in here. + // This is not meant to happen. However, since this method is not used anymore, we want it to panic + // in case somebody starts using it again in the future. This is to indicate that this is legacy code and that + // is not meant to work with new features without putting additional work into it, if ever needed. panic(fmt.Errorf("cannot shim %s to legacy ResourceMode value", addr.Resource.Resource.Mode)) } @@ -403,6 +415,8 @@ func (addr *ResourceAddress) AbsResourceInstanceAddr() addrs.AbsResourceInstance case DataResourceMode: ret.Resource.Resource.Mode = addrs.DataResourceMode default: + // This case is also covering situations when ephemeral resources are getting here. + // This shouldn't be possible, so let this panic. panic(fmt.Errorf("cannot shim %s to addrs.ResourceMode value", addr.Mode)) } diff --git a/internal/legacy/tofu/schemas.go b/internal/legacy/tofu/schemas.go index d28550b78d..32c2a01501 100644 --- a/internal/legacy/tofu/schemas.go +++ b/internal/legacy/tofu/schemas.go @@ -167,6 +167,7 @@ func loadProviderSchemas(schemas map[addrs.Provider]*ProviderSchema, config *con ) } } + // NOTE: No ephemeral resources schema for the legacy code schemas[fqn] = s @@ -271,6 +272,8 @@ func (ps *ProviderSchema) SchemaForResourceType(mode addrs.ResourceMode, typeNam case addrs.DataResourceMode: // Data resources don't have schema versions right now, since state is discarded for each refresh return ps.DataSources[typeName], 0 + case addrs.EphemeralResourceMode: + panic("ephemeral resource is not meant to be in the schema for legacy providers") default: // Shouldn't happen, because the above cases are comprehensive. return nil, 0 diff --git a/internal/legacy/tofu/state.go b/internal/legacy/tofu/state.go index a24a90c1ad..dec3df04b8 100644 --- a/internal/legacy/tofu/state.go +++ b/internal/legacy/tofu/state.go @@ -1084,6 +1084,9 @@ func (m *ModuleState) Orphans(c *configs.Module) []addrs.ResourceInstance { for _, r := range c.DataResources { inConfig[r.Addr().String()] = struct{}{} } + for _, r := range c.EphemeralResources { + log.Printf("[ERROR] ephemeral resources detected in legacy state: %s", r.Addr().String()) + } } var result []addrs.ResourceInstance diff --git a/internal/plans/action.go b/internal/plans/action.go index aef4f8a351..85175b7153 100644 --- a/internal/plans/action.go +++ b/internal/plans/action.go @@ -16,6 +16,10 @@ const ( CreateThenDelete Action = '±' Delete Action = '-' Forget Action = '.' + Open Action = '⁐' + // NOTE: Renew and Close missing on purpose. + // Those are not meant to be stored in the plan. + // Instead, we have hooks for those to show progress. ) //go:generate go run golang.org/x/tools/cmd/stringer -type Action diff --git a/internal/plans/action_string.go b/internal/plans/action_string.go index 82f68e3863..360fc5a481 100644 --- a/internal/plans/action_string.go +++ b/internal/plans/action_string.go @@ -16,6 +16,7 @@ func _() { _ = x[CreateThenDelete-177] _ = x[Delete-45] _ = x[Forget-46] + _ = x[Open-8272] } const ( @@ -24,8 +25,9 @@ const ( _Action_name_2 = "DeleteForget" _Action_name_3 = "Update" _Action_name_4 = "CreateThenDelete" - _Action_name_5 = "Read" - _Action_name_6 = "DeleteThenCreate" + _Action_name_5 = "Open" + _Action_name_6 = "Read" + _Action_name_7 = "DeleteThenCreate" ) var ( @@ -45,10 +47,12 @@ func (i Action) String() string { return _Action_name_3 case i == 177: return _Action_name_4 - case i == 8592: + case i == 8272: return _Action_name_5 - case i == 8723: + case i == 8592: return _Action_name_6 + case i == 8723: + return _Action_name_7 default: return "Action(" + strconv.FormatInt(int64(i), 10) + ")" } diff --git a/internal/plans/changes.go b/internal/plans/changes.go index 82f1ed56cd..b4902b1f0d 100644 --- a/internal/plans/changes.go +++ b/internal/plans/changes.go @@ -37,8 +37,24 @@ func NewChanges() *Changes { return &Changes{} } +// BuildChanges is a helper -- primarily intended for tests -- to build a state +// using imperative code against the StateSync type while still acting as +// an expression of type *State to assign into a containing struct. +func BuildChanges(cb func(sync *ChangesSync)) *Changes { + c := NewChanges() + cb(c.SyncWrapper()) + return c +} + func (c *Changes) Empty() bool { for _, res := range c.Resources { + // We ignore Open actions which are specific to ephemeral resources. + // A configuration containing ephemeral resources will always have changes planned, + // but if there is no other change recorded, there is no need for a prompt + // on the user. + if res.Action == Open { + continue + } if res.Action != NoOp || res.Moved() { return false } diff --git a/internal/plans/changes_test.go b/internal/plans/changes_test.go index 4ae1e5f7f0..92d8db3e38 100644 --- a/internal/plans/changes_test.go +++ b/internal/plans/changes_test.go @@ -41,6 +41,22 @@ func TestChangesEmpty(t *testing.T) { Action: Update, }, }, + // but an ephemeral resources will not impact the "emptiness" of the plan + { + Addr: addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_thing", + Name: "woot", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + PrevRunAddr: addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_thing", + Name: "woot", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + ChangeSrc: ChangeSrc{ + Action: Open, + }, + }, }, }, false, @@ -119,6 +135,28 @@ func TestChangesEmpty(t *testing.T) { }, true, }, + "ephemeral resource change": { + &Changes{ + Resources: []*ResourceInstanceChangeSrc{ + { + Addr: addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_thing", + Name: "woot", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + PrevRunAddr: addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_thing", + Name: "woot", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + ChangeSrc: ChangeSrc{ + Action: Open, + }, + }, + }, + }, + true, + }, } for name, tc := range testCases { diff --git a/internal/plans/internal/planproto/planfile.pb.go b/internal/plans/internal/planproto/planfile.pb.go index 33ca91e1d1..44c9597b42 100644 --- a/internal/plans/internal/planproto/planfile.pb.go +++ b/internal/plans/internal/planproto/planfile.pb.go @@ -88,6 +88,7 @@ const ( Action_DELETE_THEN_CREATE Action = 6 Action_CREATE_THEN_DELETE Action = 7 Action_FORGET Action = 8 + Action_OPEN Action = 9 ) // Enum value maps for Action. @@ -101,6 +102,7 @@ var ( 6: "DELETE_THEN_CREATE", 7: "CREATE_THEN_DELETE", 8: "FORGET", + 9: "OPEN", } Action_value = map[string]int32{ "NOOP": 0, @@ -111,6 +113,7 @@ var ( "DELETE_THEN_CREATE": 6, "CREATE_THEN_DELETE": 7, "FORGET": 8, + "OPEN": 9, } ) @@ -1520,49 +1523,49 @@ var file_planfile_proto_rawDesc = []byte{ 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x2a, 0x31, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x53, 0x54, 0x52, 0x4f, 0x59, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, - 0x52, 0x45, 0x46, 0x52, 0x45, 0x53, 0x48, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x2a, 0x7c, - 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4f, 0x50, - 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x08, - 0x0a, 0x04, 0x52, 0x45, 0x41, 0x44, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, - 0x54, 0x45, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x05, - 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x54, 0x48, 0x45, 0x4e, 0x5f, - 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x52, 0x45, 0x41, - 0x54, 0x45, 0x5f, 0x54, 0x48, 0x45, 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x07, - 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x08, 0x2a, 0xc8, 0x03, 0x0a, - 0x1c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x08, 0x0a, - 0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x45, 0x50, 0x4c, 0x41, - 0x43, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x54, 0x41, 0x49, 0x4e, 0x54, - 0x45, 0x44, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, - 0x42, 0x59, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x02, 0x12, 0x21, 0x0a, 0x1d, - 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, - 0x43, 0x41, 0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x12, - 0x25, 0x0a, 0x21, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, - 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x52, 0x45, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x43, 0x4f, - 0x4e, 0x46, 0x49, 0x47, 0x10, 0x04, 0x12, 0x23, 0x0a, 0x1f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, - 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x52, - 0x45, 0x50, 0x45, 0x54, 0x49, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x05, 0x12, 0x1e, 0x0a, 0x1a, 0x44, - 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x43, 0x4f, - 0x55, 0x4e, 0x54, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x06, 0x12, 0x1b, 0x0a, 0x17, 0x44, - 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x45, 0x41, - 0x43, 0x48, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x07, 0x12, 0x1c, 0x0a, 0x18, 0x44, 0x45, 0x4c, 0x45, - 0x54, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x4d, 0x4f, - 0x44, 0x55, 0x4c, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, - 0x45, 0x5f, 0x42, 0x59, 0x5f, 0x54, 0x52, 0x49, 0x47, 0x47, 0x45, 0x52, 0x53, 0x10, 0x09, 0x12, - 0x1f, 0x0a, 0x1b, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, - 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x0a, - 0x12, 0x23, 0x0a, 0x1f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, - 0x5f, 0x44, 0x45, 0x50, 0x45, 0x4e, 0x44, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x50, 0x45, 0x4e, 0x44, - 0x49, 0x4e, 0x47, 0x10, 0x0b, 0x12, 0x1d, 0x0a, 0x19, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x42, 0x45, - 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x43, 0x48, 0x45, 0x43, 0x4b, 0x5f, 0x4e, 0x45, 0x53, 0x54, - 0x45, 0x44, 0x10, 0x0d, 0x12, 0x21, 0x0a, 0x1d, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, - 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x4d, 0x4f, 0x56, 0x45, 0x5f, 0x54, - 0x41, 0x52, 0x47, 0x45, 0x54, 0x10, 0x0c, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x6f, 0x66, 0x75, 0x2f, 0x6f, - 0x70, 0x65, 0x6e, 0x74, 0x6f, 0x66, 0x75, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2f, 0x70, 0x6c, 0x61, 0x6e, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, - 0x70, 0x6c, 0x61, 0x6e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x52, 0x45, 0x46, 0x52, 0x45, 0x53, 0x48, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x2a, 0x86, + 0x01, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4f, + 0x50, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, + 0x08, 0x0a, 0x04, 0x52, 0x45, 0x41, 0x44, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, + 0x41, 0x54, 0x45, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, + 0x05, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x54, 0x48, 0x45, 0x4e, + 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x52, 0x45, + 0x41, 0x54, 0x45, 0x5f, 0x54, 0x48, 0x45, 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, + 0x07, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x08, 0x12, 0x08, 0x0a, + 0x04, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x09, 0x2a, 0xc8, 0x03, 0x0a, 0x1c, 0x52, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x41, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, 0x45, + 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x42, 0x45, + 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x54, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, + 0x16, 0x0a, 0x12, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x42, 0x59, 0x5f, 0x52, 0x45, + 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x02, 0x12, 0x21, 0x0a, 0x1d, 0x52, 0x45, 0x50, 0x4c, 0x41, + 0x43, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x4e, 0x4f, + 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x12, 0x25, 0x0a, 0x21, 0x44, 0x45, + 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, + 0x52, 0x45, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, + 0x04, 0x12, 0x23, 0x0a, 0x1f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, + 0x55, 0x53, 0x45, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x52, 0x45, 0x50, 0x45, 0x54, 0x49, + 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x05, 0x12, 0x1e, 0x0a, 0x1a, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, + 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x5f, 0x49, + 0x4e, 0x44, 0x45, 0x58, 0x10, 0x06, 0x12, 0x1b, 0x0a, 0x17, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, + 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x45, 0x41, 0x43, 0x48, 0x5f, 0x4b, 0x45, + 0x59, 0x10, 0x07, 0x12, 0x1c, 0x0a, 0x18, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x45, + 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x4d, 0x4f, 0x44, 0x55, 0x4c, 0x45, 0x10, + 0x08, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x42, 0x59, 0x5f, + 0x54, 0x52, 0x49, 0x47, 0x47, 0x45, 0x52, 0x53, 0x10, 0x09, 0x12, 0x1f, 0x0a, 0x1b, 0x52, 0x45, + 0x41, 0x44, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x49, + 0x47, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x0a, 0x12, 0x23, 0x0a, 0x1f, 0x52, + 0x45, 0x41, 0x44, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x44, 0x45, 0x50, 0x45, + 0x4e, 0x44, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x0b, + 0x12, 0x1d, 0x0a, 0x19, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, + 0x5f, 0x43, 0x48, 0x45, 0x43, 0x4b, 0x5f, 0x4e, 0x45, 0x53, 0x54, 0x45, 0x44, 0x10, 0x0d, 0x12, + 0x21, 0x0a, 0x1d, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, + 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x4d, 0x4f, 0x56, 0x45, 0x5f, 0x54, 0x41, 0x52, 0x47, 0x45, 0x54, + 0x10, 0x0c, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x6f, 0x66, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x74, 0x6f, + 0x66, 0x75, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x6c, 0x61, 0x6e, + 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x6c, 0x61, 0x6e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/plans/internal/planproto/planfile.proto b/internal/plans/internal/planproto/planfile.proto index c0f1f82fc4..c18d1a35b4 100644 --- a/internal/plans/internal/planproto/planfile.proto +++ b/internal/plans/internal/planproto/planfile.proto @@ -122,6 +122,7 @@ enum Action { DELETE_THEN_CREATE = 6; CREATE_THEN_DELETE = 7; FORGET = 8; + OPEN = 9; } // Change represents a change made to some object, transforming it from an old diff --git a/internal/plans/objchange/objchange.go b/internal/plans/objchange/objchange.go index a756720c50..d8442f6624 100644 --- a/internal/plans/objchange/objchange.go +++ b/internal/plans/objchange/objchange.go @@ -73,6 +73,16 @@ func PlannedDataResourceObject(schema *configschema.Block, config cty.Value) cty return proposedNew(schema, prior, config) } +// PlannedEphemeralResourceObject is exactly as PlannedDataResourceObject, but we +// want to have a different copy of it to emphasize the special handling of +// this type of resource. +// Ephemeral resources are not stored into the state, so every newly planned value +// is based only on the configuration and its schema. +func PlannedEphemeralResourceObject(schema *configschema.Block, config cty.Value) cty.Value { + prior := cty.UnknownVal(schema.ImpliedType()) + return proposedNew(schema, prior, config) +} + func proposedNew(schema *configschema.Block, prior, config cty.Value) cty.Value { if config.IsNull() || !config.IsKnown() { // A block config should never be null at this point. The only nullable diff --git a/internal/plans/objchange/plan_valid.go b/internal/plans/objchange/plan_valid.go index 400829b268..72ed9785ef 100644 --- a/internal/plans/objchange/plan_valid.go +++ b/internal/plans/objchange/plan_valid.go @@ -313,6 +313,9 @@ func assertPlannedValueValid(attrS *configschema.Attribute, priorV, configV, pla return assertPlannedObjectValid(attrS.NestedType, priorV, configV, plannedV, path) } + if !configV.IsNull() && plannedV.IsNull() && attrS.WriteOnly { + return errs // TODO ephemeral - check other places that might need a validation like this (part of the write-only attributes work) + } // If none of the above conditions match, the provider has made an invalid // change to this attribute. if priorV.IsNull() { diff --git a/internal/plans/planfile/tfplan.go b/internal/plans/planfile/tfplan.go index d15b2d3215..cb78cb53ba 100644 --- a/internal/plans/planfile/tfplan.go +++ b/internal/plans/planfile/tfplan.go @@ -410,6 +410,8 @@ func changeFromTfplan(rawChange *planproto.Change) (*plans.ChangeSrc, error) { ret.Action = plans.DeleteThenCreate beforeIdx = 0 afterIdx = 1 + case planproto.Action_OPEN: + ret.Action = plans.Open default: return nil, fmt.Errorf("invalid change action %s", rawChange.Action) } @@ -819,6 +821,15 @@ func changeToTfplan(change *plans.ChangeSrc) (*planproto.Change, error) { case plans.CreateThenDelete: ret.Action = planproto.Action_CREATE_THEN_DELETE ret.Values = []*planproto.DynamicValue{before, after} + case plans.Open: + ret.Action = planproto.Action_OPEN + // We need to write ephemeral resources to the plan file to be able to build + // the apply graph on `tofu apply `. + // The DiffTransformer needs the changes from the plan to be able to generate + // executable resource instance graph nodes so we are adding the ephemeral resources too. + // Even though we are writing these, the actual values of the ephemeral *must not* + // be written to the plan so set nothing here. + ret.Values = []*planproto.DynamicValue{} default: return nil, fmt.Errorf("invalid change action %s", change.Action) } diff --git a/internal/plans/planfile/tfplan_test.go b/internal/plans/planfile/tfplan_test.go index 9ef1c04af4..93a390fa4a 100644 --- a/internal/plans/planfile/tfplan_test.go +++ b/internal/plans/planfile/tfplan_test.go @@ -7,6 +7,7 @@ package planfile import ( "bytes" + "slices" "testing" "github.com/go-test/deep" @@ -173,6 +174,32 @@ func TestTFPlanRoundTrip(t *testing.T) { GeneratedConfig: "resource \\\"test_thing\\\" \\\"importing\\\" {}", }, }, + { + Addr: addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_thing", + Name: "testeph", + }.Instance(addrs.IntKey(1)).Absolute(addrs.RootModuleInstance), + PrevRunAddr: addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "test_thing", + Name: "testeph", + }.Instance(addrs.IntKey(1)).Absolute(addrs.RootModuleInstance), + ProviderAddr: addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("test"), + Module: addrs.RootModule, + }, + ChangeSrc: plans.ChangeSrc{ + Action: plans.Open, + Before: mustNewDynamicValue(cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("testing"), + }), objTy), + After: mustNewDynamicValue(cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("testing"), + }), objTy), + GeneratedConfig: "ephemeral \\\"test_thing\\\" \\\"testeph\\\" {}", + }, + }, }, }, DriftedResources: []*plans.ResourceInstanceChangeSrc{ @@ -297,6 +324,14 @@ func TestTFPlanRoundTrip(t *testing.T) { if err != nil { t.Fatal(err) } + { + // nullify the ephemeral values from the initial plan since those must be nil in the plan file + i := slices.IndexFunc(plan.Changes.Resources, func(src *plans.ResourceInstanceChangeSrc) bool { + return src.Addr.Resource.Resource.Mode == addrs.EphemeralResourceMode + }) + plan.Changes.Resources[i].After = nil + plan.Changes.Resources[i].Before = nil + } newPlan, err := readTfplan(&buf) if err != nil { diff --git a/internal/plugin/convert/schema.go b/internal/plugin/convert/schema.go index 3d06d463e3..537b468787 100644 --- a/internal/plugin/convert/schema.go +++ b/internal/plugin/convert/schema.go @@ -36,6 +36,7 @@ func ConfigSchemaToProto(b *configschema.Block) *proto.Schema_Block { Required: a.Required, Sensitive: a.Sensitive, Deprecated: a.Deprecated, + WriteOnly: a.WriteOnly, } ty, err := json.Marshal(a.Type) @@ -98,6 +99,15 @@ func ProtoToProviderSchema(s *proto.Schema) providers.Schema { } } +// ProtoToEphemeralProviderSchema takes a proto.Schema and converts it to a providers.Schema +// marking it as being able to work with ephemeral values. +func ProtoToEphemeralProviderSchema(s *proto.Schema) providers.Schema { + ret := ProtoToProviderSchema(s) + ret.Block.Ephemeral = true + + return ret +} + // ProtoToConfigSchema takes the Schema_Block from a grpc response and converts it // to a tofu *configschema.Block. func ProtoToConfigSchema(b *proto.Schema_Block) *configschema.Block { @@ -119,6 +129,7 @@ func ProtoToConfigSchema(b *proto.Schema_Block) *configschema.Block { Computed: a.Computed, Sensitive: a.Sensitive, Deprecated: a.Deprecated, + WriteOnly: a.WriteOnly, } if err := json.Unmarshal(a.Type, &attr.Type); err != nil { diff --git a/internal/plugin/grpc_provider.go b/internal/plugin/grpc_provider.go index 36e4c6f382..58a65ddc5c 100644 --- a/internal/plugin/grpc_provider.go +++ b/internal/plugin/grpc_provider.go @@ -40,7 +40,8 @@ var clientCapabilities = &proto.ClientCapabilities{ // satisfy the request. Setting this means that we need to be prepared // for there to be a "deferred" object in the response from various // other provider RPC functions. - DeferralAllowed: true, + DeferralAllowed: true, + WriteOnlyAttributesAllowed: true, } func (p *GRPCProviderPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { @@ -121,6 +122,7 @@ func (p *GRPCProvider) GetProviderSchema(ctx context.Context) (resp providers.Ge resp.ResourceTypes = make(map[string]providers.Schema) resp.DataSources = make(map[string]providers.Schema) + resp.EphemeralResources = make(map[string]providers.Schema) resp.Functions = make(map[string]providers.FunctionSpec) // Some providers may generate quite large schemas, and the internal default @@ -149,7 +151,9 @@ func (p *GRPCProvider) GetProviderSchema(ctx context.Context) (resp providers.Ge return resp } - resp.Provider = convert.ProtoToProviderSchema(protoResp.Provider) + // We want to allow "provider" blocks to work with ephemeral variables, so we + // just mark its schema as able to get such values. + resp.Provider = convert.ProtoToEphemeralProviderSchema(protoResp.Provider) if protoResp.ProviderMeta == nil { logger.Debug("No provider meta schema returned") } else { @@ -164,6 +168,11 @@ func (p *GRPCProvider) GetProviderSchema(ctx context.Context) (resp providers.Ge resp.DataSources[name] = convert.ProtoToProviderSchema(data) } + for name, data := range protoResp.EphemeralResourceSchemas { + // Ephemeral resources should be able to work with ephemeral values by design. + resp.EphemeralResources[name] = convert.ProtoToEphemeralProviderSchema(data) + } + for name, fn := range protoResp.Functions { resp.Functions[name] = convert.ProtoToFunctionSpec(fn) } @@ -253,8 +262,9 @@ func (p *GRPCProvider) ValidateResourceConfig(ctx context.Context, r providers.V } protoReq := &proto.ValidateResourceTypeConfig_Request{ - TypeName: r.TypeName, - Config: &proto.DynamicValue{Msgpack: mp}, + TypeName: r.TypeName, + Config: &proto.DynamicValue{Msgpack: mp}, + ClientCapabilities: clientCapabilities, } protoResp, err := p.client.ValidateResourceTypeConfig(ctx, protoReq) @@ -302,6 +312,41 @@ func (p *GRPCProvider) ValidateDataResourceConfig(ctx context.Context, r provide return resp } +func (p *GRPCProvider) ValidateEphemeralConfig(ctx context.Context, r providers.ValidateEphemeralConfigRequest) (resp providers.ValidateEphemeralConfigResponse) { + logger.Trace("GRPCProvider: ValidateEphemeralConfig") + + schema := p.GetProviderSchema(ctx) + if schema.Diagnostics.HasErrors() { + resp.Diagnostics = schema.Diagnostics + return resp + } + + ephemeralSchema, ok := schema.EphemeralResources[r.TypeName] + if !ok { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("unknown ephemeral resource %q", r.TypeName)) + return resp + } + + mp, err := msgpack.Marshal(r.Config, ephemeralSchema.Block.ImpliedType()) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + + protoReq := &proto.ValidateEphemeralResourceConfig_Request{ + TypeName: r.TypeName, + Config: &proto.DynamicValue{Msgpack: mp}, + } + + protoResp, err := p.client.ValidateEphemeralResourceConfig(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + return resp +} + func (p *GRPCProvider) UpgradeResourceState(ctx context.Context, r providers.UpgradeResourceStateRequest) (resp providers.UpgradeResourceStateResponse) { logger.Trace("GRPCProvider: UpgradeResourceState") @@ -794,6 +839,101 @@ func (p *GRPCProvider) ReadDataSource(ctx context.Context, r providers.ReadDataS return resp } +func (p *GRPCProvider) OpenEphemeralResource(ctx context.Context, r providers.OpenEphemeralResourceRequest) (resp providers.OpenEphemeralResourceResponse) { + logger.Trace("GRPCProvider: OpenEphemeralResource") + + schema := p.GetProviderSchema(ctx) + resp.Diagnostics = schema.Diagnostics + if resp.Diagnostics.HasErrors() { + return resp + } + + ephemeralResourceSchema, ok := schema.EphemeralResources[r.TypeName] + if !ok { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("unknown ephemeral resource %q", r.TypeName)) + return resp + } + + config, err := msgpack.Marshal(r.Config, ephemeralResourceSchema.Block.ImpliedType()) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + + protoReq := &proto.OpenEphemeralResource_Request{ + TypeName: r.TypeName, + Config: &proto.DynamicValue{ + Msgpack: config, + }, + ClientCapabilities: clientCapabilities, + } + + protoResp, err := p.client.OpenEphemeralResource(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + + result, err := decodeDynamicValue(protoResp.Result, ephemeralResourceSchema.Block.ImpliedType()) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + resp.Result = result + resp.Private = protoResp.Private + if protoResp.RenewAt != nil { + renewAt := protoResp.RenewAt.AsTime() + resp.RenewAt = &renewAt + } + if protoDeferred := protoResp.Deferred; protoDeferred != nil { + resp.Deferred = &providers.EphemeralResourceDeferred{DeferralReason: convert.DeferralReasonFromProto(protoDeferred.Reason)} + } + + return resp +} + +func (p *GRPCProvider) RenewEphemeralResource(ctx context.Context, r providers.RenewEphemeralResourceRequest) (resp providers.RenewEphemeralResourceResponse) { + logger.Trace("GRPCProvider: RenewEphemeralResource") + + protoReq := &proto.RenewEphemeralResource_Request{ + TypeName: r.TypeName, + Private: r.Private, + } + + protoResp, err := p.client.RenewEphemeralResource(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + resp.Private = protoResp.Private + if protoResp.RenewAt != nil { + renewAt := protoResp.RenewAt.AsTime() + resp.RenewAt = &renewAt + } + + return resp +} + +func (p *GRPCProvider) CloseEphemeralResource(ctx context.Context, r providers.CloseEphemeralResourceRequest) (resp providers.CloseEphemeralResourceResponse) { + logger.Trace("GRPCProvider: CloseEphemeralResource") + + protoReq := &proto.CloseEphemeralResource_Request{ + TypeName: r.TypeName, + Private: r.Private, + } + + protoResp, err := p.client.CloseEphemeralResource(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + + return resp +} + func (p *GRPCProvider) GetFunctions(ctx context.Context) (resp providers.GetFunctionsResponse) { logger.Trace("GRPCProvider: GetFunctions") diff --git a/internal/plugin/grpc_provider_test.go b/internal/plugin/grpc_provider_test.go index e15fe90d43..f4b0f3cdbb 100644 --- a/internal/plugin/grpc_provider_test.go +++ b/internal/plugin/grpc_provider_test.go @@ -8,12 +8,16 @@ package plugin import ( "bytes" "fmt" + "slices" + "strings" "testing" + "time" "github.com/davecgh/go-spew/spew" "github.com/google/go-cmp/cmp" "github.com/zclconf/go-cty/cty" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/legacy/hcl2shim" @@ -96,6 +100,20 @@ func providerProtoSchema() *proto.GetProviderSchema_Response { }, }, }, + EphemeralResourceSchemas: map[string]*proto.Schema{ + "eph": { + Version: 1, + Block: &proto.Schema_Block{ + Attributes: []*proto.Schema_Attribute{ + { + Name: "attr", + Type: []byte(`"string"`), + Required: true, + }, + }, + }, + }, + }, Functions: map[string]*proto.Function{ "fn": &proto.Function{ Parameters: []*proto.Function_Parameter{{ @@ -125,6 +143,22 @@ func TestGRPCProvider_GetSchema(t *testing.T) { resp := p.GetProviderSchema(t.Context()) checkDiags(t, resp.Diagnostics) + + { // check ephemeral attribute of the schema blocks + if !resp.Provider.Block.Ephemeral { + t.Errorf("provider.Block.Ephemeral meant to be true") + } + checkResources := func(t *testing.T, r map[string]providers.Schema, want bool) { + for typ, schema := range r { + if schema.Block.Ephemeral != want { + t.Errorf("expected resource %q to have ephemeral as %t", typ, want) + } + } + } + checkResources(t, resp.ResourceTypes, false) + checkResources(t, resp.DataSources, false) + checkResources(t, resp.EphemeralResources, true) + } } // Ensure that gRPC errors are returned early. @@ -327,6 +361,25 @@ func TestGRPCProvider_ValidateDataSourceConfig(t *testing.T) { checkDiags(t, resp.Diagnostics) } +func TestGRPCProvider_ValidateEphemeralResourceConfig(t *testing.T) { + client := mockProviderClient(t) + p := &GRPCProvider{ + client: client, + } + + client.EXPECT().ValidateEphemeralResourceConfig( + gomock.Any(), + gomock.Any(), + ).Return(&proto.ValidateEphemeralResourceConfig_Response{}, nil) + + cfg := hcl2shim.HCL2ValueFromConfigValue(map[string]interface{}{"attr": "value"}) + resp := p.ValidateEphemeralConfig(t.Context(), providers.ValidateEphemeralConfigRequest{ + TypeName: "eph", + Config: cfg, + }) + checkDiags(t, resp.Diagnostics) +} + func TestGRPCProvider_UpgradeResourceState(t *testing.T) { client := mockProviderClient(t) p := &GRPCProvider{ @@ -978,6 +1031,127 @@ func TestGRPCProvider_ReadDataSourceJSON(t *testing.T) { } } +func TestGRPCProvider_OpenEphemeralResource(t *testing.T) { + t.Run("success", func(t *testing.T) { + client := mockProviderClient(t) + p := &GRPCProvider{ + client: client, + } + + future := time.Now().Add(time.Minute) + client.EXPECT().OpenEphemeralResource( + gomock.Any(), + gomock.Any(), + ).Return(&proto.OpenEphemeralResource_Response{ + Result: &proto.DynamicValue{ + Msgpack: []byte("\x81\xa4attr\xa3bar"), + }, + Private: []byte("private data"), + RenewAt: timestamppb.New(future), + Deferred: &proto.Deferred{ + Reason: proto.Deferred_RESOURCE_CONFIG_UNKNOWN, + }, + }, nil) + + resp := p.OpenEphemeralResource(t.Context(), providers.OpenEphemeralResourceRequest{ + TypeName: "eph", + Config: cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("foo"), + }), + }) + + checkDiags(t, resp.Diagnostics) + + expected := cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("bar"), + }) + if diff := cmp.Diff(expected, resp.Result, typeComparer, valueComparer, equateEmpty); diff != "" { + t.Fatalf("expected to have no diff between the expected result and result from the openEphemeral. got: %s", diff) + } + if resp.RenewAt == nil || !future.Equal(*resp.RenewAt) { + t.Fatalf("unexpected renewAt. got: %s, want %s", resp.RenewAt, future) + } + if got, want := resp.Private, []byte("private data"); !slices.Equal(got, want) { + t.Fatalf("unexpected private data. got: %q, want %q", got, want) + } + { + if resp.Deferred == nil { + t.Fatal("expected to have a deferred object but got none") + } + if got, want := resp.Deferred.DeferralReason, providers.DeferredBecauseResourceConfigUnknown; got != want { + t.Fatalf("unexpected deferred reason. got: %d, want %d", got, want) + } + } + }) + t.Run("requested type is not in schema", func(t *testing.T) { + client := mockProviderClient(t) + p := &GRPCProvider{ + client: client, + } + + resp := p.OpenEphemeralResource(t.Context(), providers.OpenEphemeralResourceRequest{ + TypeName: "non_existing", + Config: cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("foo"), + }), + }) + checkDiagsHasError(t, resp.Diagnostics) + if got, want := resp.Diagnostics.Err().Error(), `unknown ephemeral resource "non_existing"`; !strings.Contains(got, want) { + t.Fatalf("diagnostis does not contain the expected content. got: %s; want: %s", got, want) + } + }) +} + +func TestGRPCProvider_RenewEphemeralResource(t *testing.T) { + ctrl := gomock.NewController(t) + client := mockproto.NewMockProviderClient(ctrl) + p := &GRPCProvider{ + client: client, + } + + future := time.Now().Add(time.Minute) + client.EXPECT().RenewEphemeralResource( + gomock.Any(), + gomock.Any(), + ).Return(&proto.RenewEphemeralResource_Response{ + Private: []byte("private data new"), + RenewAt: timestamppb.New(future), + }, nil) + + resp := p.RenewEphemeralResource(t.Context(), providers.RenewEphemeralResourceRequest{ + TypeName: "eph", + }) + + checkDiags(t, resp.Diagnostics) + + if resp.RenewAt == nil || !future.Equal(*resp.RenewAt) { + t.Fatalf("unexpected renewAt. got: %s, want %s", resp.RenewAt, future) + } + + if got, want := resp.Private, []byte("private data new"); !slices.Equal(got, want) { + t.Fatalf("unexpected private data. got: %q, want %q", got, want) + } +} + +func TestGRPCProvider_CloseEphemeralResource(t *testing.T) { + ctrl := gomock.NewController(t) + client := mockproto.NewMockProviderClient(ctrl) + p := &GRPCProvider{ + client: client, + } + + client.EXPECT().CloseEphemeralResource( + gomock.Any(), + gomock.Any(), + ).Return(&proto.CloseEphemeralResource_Response{}, nil) + + resp := p.CloseEphemeralResource(t.Context(), providers.CloseEphemeralResourceRequest{ + TypeName: "eph", + }) + + checkDiags(t, resp.Diagnostics) +} + func TestGRPCProvider_CallFunction(t *testing.T) { client := mockProviderClient(t) p := &GRPCProvider{ diff --git a/internal/plugin6/convert/schema.go b/internal/plugin6/convert/schema.go index d4f2ca8f21..3ca95e9902 100644 --- a/internal/plugin6/convert/schema.go +++ b/internal/plugin6/convert/schema.go @@ -37,6 +37,7 @@ func ConfigSchemaToProto(b *configschema.Block) *proto.Schema_Block { Required: a.Required, Sensitive: a.Sensitive, Deprecated: a.Deprecated, + WriteOnly: a.WriteOnly, } if a.Type != cty.NilType { @@ -104,6 +105,15 @@ func ProtoToProviderSchema(s *proto.Schema) providers.Schema { } } +// ProtoToEphemeralProviderSchema takes a proto.Schema and converts it to a providers.Schema +// marking it as being able to work with ephemeral values. +func ProtoToEphemeralProviderSchema(s *proto.Schema) providers.Schema { + ret := ProtoToProviderSchema(s) + ret.Block.Ephemeral = true + + return ret +} + // ProtoToConfigSchema takes the Schema_Block from a grpc response and converts it // to a tofu *configschema.Block. func ProtoToConfigSchema(b *proto.Schema_Block) *configschema.Block { @@ -125,6 +135,7 @@ func ProtoToConfigSchema(b *proto.Schema_Block) *configschema.Block { Computed: a.Computed, Sensitive: a.Sensitive, Deprecated: a.Deprecated, + WriteOnly: a.WriteOnly, } if a.Type != nil { @@ -215,6 +226,7 @@ func protoObjectToConfigSchema(b *proto.Schema_Object) *configschema.Object { Computed: a.Computed, Sensitive: a.Sensitive, Deprecated: a.Deprecated, + WriteOnly: a.WriteOnly, } if a.Type != nil { diff --git a/internal/plugin6/grpc_provider.go b/internal/plugin6/grpc_provider.go index c417c2ba5b..bb0bdcb406 100644 --- a/internal/plugin6/grpc_provider.go +++ b/internal/plugin6/grpc_provider.go @@ -40,7 +40,8 @@ var clientCapabilities = &proto6.ClientCapabilities{ // satisfy the request. Setting this means that we need to be prepared // for there to be a "deferred" object in the response from various // other provider RPC functions. - DeferralAllowed: true, + DeferralAllowed: true, + WriteOnlyAttributesAllowed: true, } func (p *GRPCProviderPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { @@ -95,6 +96,7 @@ type GRPCProvider struct { var _ providers.Interface = new(GRPCProvider) +// TODO ephemeral - double check all of the usages of this to be sure that the block.ephemeral for ephemeral resources is used accordingly. func (p *GRPCProvider) GetProviderSchema(ctx context.Context) (resp providers.GetProviderSchemaResponse) { logger.Trace("GRPCProvider.v6: GetProviderSchema") p.mu.Lock() @@ -121,6 +123,7 @@ func (p *GRPCProvider) GetProviderSchema(ctx context.Context) (resp providers.Ge resp.ResourceTypes = make(map[string]providers.Schema) resp.DataSources = make(map[string]providers.Schema) + resp.EphemeralResources = make(map[string]providers.Schema) resp.Functions = make(map[string]providers.FunctionSpec) // Some providers may generate quite large schemas, and the internal default @@ -149,7 +152,9 @@ func (p *GRPCProvider) GetProviderSchema(ctx context.Context) (resp providers.Ge return resp } - resp.Provider = convert.ProtoToProviderSchema(protoResp.Provider) + // We want to allow "provider" blocks to work with ephemeral variables, so we + // just mark its schema as able to get such values. + resp.Provider = convert.ProtoToEphemeralProviderSchema(protoResp.Provider) if protoResp.ProviderMeta == nil { logger.Debug("No provider meta schema returned") } else { @@ -168,6 +173,11 @@ func (p *GRPCProvider) GetProviderSchema(ctx context.Context) (resp providers.Ge resp.Functions[name] = convert.ProtoToFunctionSpec(fn) } + for name, res := range protoResp.EphemeralResourceSchemas { + // Ephemeral resources should be able to work with ephemeral values by design. + resp.EphemeralResources[name] = convert.ProtoToEphemeralProviderSchema(res) + } + if protoResp.ServerCapabilities != nil { resp.ServerCapabilities.PlanDestroy = protoResp.ServerCapabilities.PlanDestroy resp.ServerCapabilities.GetProviderSchemaOptional = protoResp.ServerCapabilities.GetProviderSchemaOptional @@ -246,8 +256,9 @@ func (p *GRPCProvider) ValidateResourceConfig(ctx context.Context, r providers.V } protoReq := &proto6.ValidateResourceConfig_Request{ - TypeName: r.TypeName, - Config: &proto6.DynamicValue{Msgpack: mp}, + TypeName: r.TypeName, + Config: &proto6.DynamicValue{Msgpack: mp}, + ClientCapabilities: clientCapabilities, } protoResp, err := p.client.ValidateResourceConfig(ctx, protoReq) @@ -295,6 +306,41 @@ func (p *GRPCProvider) ValidateDataResourceConfig(ctx context.Context, r provide return resp } +func (p *GRPCProvider) ValidateEphemeralConfig(ctx context.Context, r providers.ValidateEphemeralConfigRequest) (resp providers.ValidateEphemeralConfigResponse) { + logger.Trace("GRPCProvider.v6: ValidateEphemeralConfig") + + schema := p.GetProviderSchema(ctx) + if schema.Diagnostics.HasErrors() { + resp.Diagnostics = schema.Diagnostics + return resp + } + + ephemeralSchema, ok := schema.EphemeralResources[r.TypeName] + if !ok { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("unknown ephemeral resource %q", r.TypeName)) + return resp + } + + mp, err := msgpack.Marshal(r.Config, ephemeralSchema.Block.ImpliedType()) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + + protoReq := &proto6.ValidateEphemeralResourceConfig_Request{ + TypeName: r.TypeName, + Config: &proto6.DynamicValue{Msgpack: mp}, + } + + protoResp, err := p.client.ValidateEphemeralResourceConfig(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + return resp +} + func (p *GRPCProvider) UpgradeResourceState(ctx context.Context, r providers.UpgradeResourceStateRequest) (resp providers.UpgradeResourceStateResponse) { logger.Trace("GRPCProvider.v6: UpgradeResourceState") @@ -783,6 +829,101 @@ func (p *GRPCProvider) ReadDataSource(ctx context.Context, r providers.ReadDataS return resp } +func (p *GRPCProvider) OpenEphemeralResource(ctx context.Context, r providers.OpenEphemeralResourceRequest) (resp providers.OpenEphemeralResourceResponse) { + logger.Trace("GRPCProvider.v6: OpenEphemeralResource") + + schema := p.GetProviderSchema(ctx) + resp.Diagnostics = schema.Diagnostics + if resp.Diagnostics.HasErrors() { + return resp + } + + ephemeralResourceSchema, ok := schema.EphemeralResources[r.TypeName] + if !ok { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("unknown ephemeral resource %q", r.TypeName)) + return resp + } + + config, err := msgpack.Marshal(r.Config, ephemeralResourceSchema.Block.ImpliedType()) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + + protoReq := &proto6.OpenEphemeralResource_Request{ + TypeName: r.TypeName, + Config: &proto6.DynamicValue{ + Msgpack: config, + }, + ClientCapabilities: clientCapabilities, + } + + protoResp, err := p.client.OpenEphemeralResource(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + + result, err := decodeDynamicValue(protoResp.Result, ephemeralResourceSchema.Block.ImpliedType()) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + resp.Result = result + resp.Private = protoResp.Private + if protoResp.RenewAt != nil { + renewAt := protoResp.RenewAt.AsTime() + resp.RenewAt = &renewAt + } + if protoDeferred := protoResp.Deferred; protoDeferred != nil { + resp.Deferred = &providers.EphemeralResourceDeferred{DeferralReason: convert.DeferralReasonFromProto(protoDeferred.Reason)} + } + + return resp +} + +func (p *GRPCProvider) RenewEphemeralResource(ctx context.Context, r providers.RenewEphemeralResourceRequest) (resp providers.RenewEphemeralResourceResponse) { + logger.Trace("GRPCProvider.v6: RenewEphemeralResource") + + protoReq := &proto6.RenewEphemeralResource_Request{ + TypeName: r.TypeName, + Private: r.Private, + } + + protoResp, err := p.client.RenewEphemeralResource(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + resp.Private = protoResp.Private + if protoResp.RenewAt != nil { + renewAt := protoResp.RenewAt.AsTime() + resp.RenewAt = &renewAt + } + + return resp +} + +func (p *GRPCProvider) CloseEphemeralResource(ctx context.Context, r providers.CloseEphemeralResourceRequest) (resp providers.CloseEphemeralResourceResponse) { + logger.Trace("GRPCProvider.v6: CloseEphemeralResource") + + protoReq := &proto6.CloseEphemeralResource_Request{ + TypeName: r.TypeName, + Private: r.Private, + } + + protoResp, err := p.client.CloseEphemeralResource(ctx, protoReq) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err)) + return resp + } + resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics)) + + return resp +} + func (p *GRPCProvider) GetFunctions(ctx context.Context) (resp providers.GetFunctionsResponse) { logger.Trace("GRPCProvider6: GetFunctions") diff --git a/internal/plugin6/grpc_provider_test.go b/internal/plugin6/grpc_provider_test.go index 5111ba7a13..1934f6a678 100644 --- a/internal/plugin6/grpc_provider_test.go +++ b/internal/plugin6/grpc_provider_test.go @@ -8,13 +8,17 @@ package plugin6 import ( "bytes" "fmt" + "slices" + "strings" "testing" + "time" "github.com/davecgh/go-spew/spew" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/zclconf/go-cty/cty" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/legacy/hcl2shim" @@ -103,6 +107,20 @@ func providerProtoSchema() *proto.GetProviderSchema_Response { }, }, }, + EphemeralResourceSchemas: map[string]*proto.Schema{ + "eph": { + Version: 1, + Block: &proto.Schema_Block{ + Attributes: []*proto.Schema_Attribute{ + { + Name: "attr", + Type: []byte(`"string"`), + Required: true, + }, + }, + }, + }, + }, Functions: map[string]*proto.Function{ "fn": &proto.Function{ Parameters: []*proto.Function_Parameter{{ @@ -132,6 +150,22 @@ func TestGRPCProvider_GetSchema(t *testing.T) { resp := p.GetProviderSchema(t.Context()) checkDiags(t, resp.Diagnostics) + + { // check ephemeral attribute of the schema blocks + if !resp.Provider.Block.Ephemeral { + t.Errorf("provider.Block.Ephemeral meant to be true") + } + checkResources := func(t *testing.T, r map[string]providers.Schema, want bool) { + for typ, schema := range r { + if schema.Block.Ephemeral != want { + t.Errorf("expected resource %q to have ephemeral as %t", typ, want) + } + } + } + checkResources(t, resp.ResourceTypes, false) + checkResources(t, resp.DataSources, false) + checkResources(t, resp.EphemeralResources, true) + } } // Ensure that gRPC errors are returned early. @@ -334,6 +368,25 @@ func TestGRPCProvider_ValidateDataResourceConfig(t *testing.T) { checkDiags(t, resp.Diagnostics) } +func TestGRPCProvider_ValidateEphemeralResourceConfig(t *testing.T) { + client := mockProviderClient(t) + p := &GRPCProvider{ + client: client, + } + + client.EXPECT().ValidateEphemeralResourceConfig( + gomock.Any(), + gomock.Any(), + ).Return(&proto.ValidateEphemeralResourceConfig_Response{}, nil) + + cfg := hcl2shim.HCL2ValueFromConfigValue(map[string]interface{}{"attr": "value"}) + resp := p.ValidateEphemeralConfig(t.Context(), providers.ValidateEphemeralConfigRequest{ + TypeName: "eph", + Config: cfg, + }) + checkDiags(t, resp.Diagnostics) +} + func TestGRPCProvider_UpgradeResourceState(t *testing.T) { client := mockProviderClient(t) p := &GRPCProvider{ @@ -984,6 +1037,127 @@ func TestGRPCProvider_ReadDataSourceJSON(t *testing.T) { } } +func TestGRPCProvider_OpenEphemeralResource(t *testing.T) { + t.Run("success", func(t *testing.T) { + client := mockProviderClient(t) + p := &GRPCProvider{ + client: client, + } + + future := time.Now().Add(time.Minute) + client.EXPECT().OpenEphemeralResource( + gomock.Any(), + gomock.Any(), + ).Return(&proto.OpenEphemeralResource_Response{ + Result: &proto.DynamicValue{ + Msgpack: []byte("\x81\xa4attr\xa3bar"), + }, + Private: []byte("private data"), + RenewAt: timestamppb.New(future), + Deferred: &proto.Deferred{ + Reason: proto.Deferred_RESOURCE_CONFIG_UNKNOWN, + }, + }, nil) + + resp := p.OpenEphemeralResource(t.Context(), providers.OpenEphemeralResourceRequest{ + TypeName: "eph", + Config: cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("foo"), + }), + }) + + checkDiags(t, resp.Diagnostics) + + expected := cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("bar"), + }) + if diff := cmp.Diff(expected, resp.Result, typeComparer, valueComparer, equateEmpty); diff != "" { + t.Fatalf("expected to have no diff between the expected result and result from the openEphemeral. got: %s", diff) + } + if resp.RenewAt == nil || !future.Equal(*resp.RenewAt) { + t.Fatalf("unexpected renewAt. got: %s, want %s", resp.RenewAt, future) + } + if got, want := resp.Private, []byte("private data"); !slices.Equal(got, want) { + t.Fatalf("unexpected private data. got: %q, want %q", got, want) + } + { + if resp.Deferred == nil { + t.Fatal("expected to have a deferred object but got none") + } + if got, want := resp.Deferred.DeferralReason, providers.DeferredBecauseResourceConfigUnknown; got != want { + t.Fatalf("unexpected deferred reason. got: %d, want %d", got, want) + } + } + }) + t.Run("requested type is not in schema", func(t *testing.T) { + client := mockProviderClient(t) + p := &GRPCProvider{ + client: client, + } + + resp := p.OpenEphemeralResource(t.Context(), providers.OpenEphemeralResourceRequest{ + TypeName: "non_existing", + Config: cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("foo"), + }), + }) + checkDiagsHasError(t, resp.Diagnostics) + if got, want := resp.Diagnostics.Err().Error(), `unknown ephemeral resource "non_existing"`; !strings.Contains(got, want) { + t.Fatalf("diagnostis does not contain the expected content. got: %s; want: %s", got, want) + } + }) +} + +func TestGRPCProvider_RenewEphemeralResource(t *testing.T) { + ctrl := gomock.NewController(t) + client := mockproto.NewMockProviderClient(ctrl) + p := &GRPCProvider{ + client: client, + } + + future := time.Now().Add(time.Minute) + client.EXPECT().RenewEphemeralResource( + gomock.Any(), + gomock.Any(), + ).Return(&proto.RenewEphemeralResource_Response{ + Private: []byte("private data new"), + RenewAt: timestamppb.New(future), + }, nil) + + resp := p.RenewEphemeralResource(t.Context(), providers.RenewEphemeralResourceRequest{ + TypeName: "eph", + }) + + checkDiags(t, resp.Diagnostics) + + if resp.RenewAt == nil || !future.Equal(*resp.RenewAt) { + t.Fatalf("unexpected renewAt. got: %s, want %s", resp.RenewAt, future) + } + + if got, want := resp.Private, []byte("private data new"); !slices.Equal(got, want) { + t.Fatalf("unexpected private data. got: %q, want %q", got, want) + } +} + +func TestGRPCProvider_CloseEphemeralResource(t *testing.T) { + ctrl := gomock.NewController(t) + client := mockproto.NewMockProviderClient(ctrl) + p := &GRPCProvider{ + client: client, + } + + client.EXPECT().CloseEphemeralResource( + gomock.Any(), + gomock.Any(), + ).Return(&proto.CloseEphemeralResource_Response{}, nil) + + resp := p.CloseEphemeralResource(t.Context(), providers.CloseEphemeralResourceRequest{ + TypeName: "eph", + }) + + checkDiags(t, resp.Diagnostics) +} + func TestGRPCProvider_CallFunction(t *testing.T) { client := mockProviderClient(t) p := &GRPCProvider{ diff --git a/internal/provider-simple-v6/provider.go b/internal/provider-simple-v6/provider.go index 2b0d0572d5..e9bbb496ff 100644 --- a/internal/provider-simple-v6/provider.go +++ b/internal/provider-simple-v6/provider.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/opentofu/opentofu/internal/configs/configschema" @@ -37,18 +38,46 @@ func Provider() providers.Interface { }, }, } + // Only managed resource should have write-only arguments. + withWriteOnlyAttribute := func(s providers.Schema) providers.Schema { + b := *s.Block + + b.Attributes["value_wo"] = &configschema.Attribute{ + Optional: true, + Type: cty.String, + WriteOnly: true, + } + return providers.Schema{Block: &b} + } return simple{ schema: providers.GetProviderSchemaResponse{ Provider: providers.Schema{ - Block: nil, + // The "i_depend_on" field is just a simple configuration attribute of the provider + // to allow creation of dependencies between a resources from a previously + // initialized provider and this provider. + // The "i_depend_on" field is having no functionality behind, in the provider context, + // but it's just a way for the "provider" block to create depedencies + // to other blocks. + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "i_depend_on": { + Type: cty.String, + Description: "Non-functional configuration attribute of the provider. This is meant to be used only to create depedencies of other resources to the provider block", + Optional: true, + }, + }, + }, }, ResourceTypes: map[string]providers.Schema{ - "simple_resource": simpleResource, + "simple_resource": withWriteOnlyAttribute(simpleResource), }, DataSources: map[string]providers.Schema{ "simple_resource": simpleResource, }, + EphemeralResources: map[string]providers.Schema{ + "simple_resource": simpleResource, + }, ServerCapabilities: providers.ServerCapabilities{ PlanDestroy: true, }, @@ -72,6 +101,10 @@ func (s simple) ValidateDataResourceConfig(_ context.Context, req providers.Vali return resp } +func (s simple) ValidateEphemeralConfig(context.Context, providers.ValidateEphemeralConfigRequest) (resp providers.ValidateEphemeralConfigResponse) { + return resp +} + func (s simple) MoveResourceState(_ context.Context, req providers.MoveResourceStateRequest) providers.MoveResourceStateResponse { var resp providers.MoveResourceStateResponse val, err := ctyjson.Unmarshal(req.SourceStateJSON, s.schema.ResourceTypes["simple_resource"].Block.ImpliedType()) @@ -122,6 +155,16 @@ func (s simple) PlanResourceChange(_ context.Context, req providers.PlanResource if !ok { m["id"] = cty.UnknownVal(cty.String) } + // TODO ephemeral - remove this line after work will be done on write-only arguments. + // The problem now is that the value sent to ApplyResourceChange is always null as returned by the plan call. + // When the work on write-only arguments will be done, OpenTofu should send the actual value to + // the ApplyResourceChange too. + // To confirm that everything is ok, by removing this "waitIfRequested" call from here, theTestEphemeralWorkflowAndOutput + // should still work correctly without any warn logs in the test output + waitIfRequested(m) + + // Simulate what the terraform-plugin-go should do. Nullify the write-only attributes. + m["value_wo"] = cty.NullVal(cty.String) resp.PlannedState = cty.ObjectVal(m) return resp @@ -143,6 +186,10 @@ func (s simple) ApplyResourceChange(_ context.Context, req providers.ApplyResour if !ok { m["id"] = cty.StringVal(time.Now().String()) } + waitIfRequested(m) + + // Simulate what the terraform-plugin-go should do. Nullify the write-only attributes. + m["value_wo"] = cty.NullVal(cty.String) resp.NewState = cty.ObjectVal(m) return resp @@ -160,6 +207,29 @@ func (s simple) ReadDataSource(_ context.Context, req providers.ReadDataSourceRe return resp } +func (s simple) OpenEphemeralResource(_ context.Context, req providers.OpenEphemeralResourceRequest) (resp providers.OpenEphemeralResourceResponse) { + m := req.Config.AsValueMap() + m["id"] = cty.StringVal("static-ephemeral-id") + if v, ok := m["value"]; ok && !v.IsNull() && strings.Contains(v.AsString(), "with-renew") { + t := time.Now().Add(200 * time.Millisecond) + resp.RenewAt = &t + } + resp.Result = cty.ObjectVal(m) + resp.Private = []byte("static private data") + return resp +} + +func (s simple) RenewEphemeralResource(_ context.Context, req providers.RenewEphemeralResourceRequest) (resp providers.RenewEphemeralResourceResponse) { + resp.Private = []byte(fmt.Sprintf("%s - renew", req.Private)) + t := time.Now().Add(200 * time.Millisecond) + resp.RenewAt = &t + return resp +} + +func (s simple) CloseEphemeralResource(context.Context, providers.CloseEphemeralResourceRequest) (resp providers.CloseEphemeralResourceResponse) { + return resp +} + func (s simple) GetFunctions(context.Context) providers.GetFunctionsResponse { panic("Not Implemented") } @@ -171,3 +241,12 @@ func (s simple) CallFunction(_ context.Context, r providers.CallFunctionRequest) func (s simple) Close(_ context.Context) error { return nil } + +func waitIfRequested(m map[string]cty.Value) { + // This is a special case that can be used together with ephemeral resources to be able to test the renewal process. + // When the "value" attribute of the resource is containing "with-renew" it will return later to allow + // the ephemeral resource to call renew at least once. Check also OpenEphemeralResource. + if v, ok := m["value_wo"]; ok && !v.IsNull() && strings.Contains(v.AsString(), "with-renew") { + <-time.After(time.Second) + } +} diff --git a/internal/provider-simple/provider.go b/internal/provider-simple/provider.go index c499023812..11a6527158 100644 --- a/internal/provider-simple/provider.go +++ b/internal/provider-simple/provider.go @@ -9,6 +9,7 @@ package simple import ( "context" "errors" + "strings" "time" "github.com/opentofu/opentofu/internal/configs/configschema" @@ -36,18 +37,46 @@ func Provider() providers.Interface { }, }, } + // Only managed resource should have write-only arguments. + withWriteOnlyAttribute := func(s providers.Schema) providers.Schema { + b := *s.Block + + b.Attributes["value_wo"] = &configschema.Attribute{ + Optional: true, + Type: cty.String, + WriteOnly: true, + } + return providers.Schema{Block: &b} + } return simple{ schema: providers.GetProviderSchemaResponse{ Provider: providers.Schema{ - Block: nil, + // The "i_depend_on" field is just a simple configuration attribute of the provider + // to allow creation of dependencies between a resources from a previously + // initialized provider and this provider. + // The "i_depend_on" field is having no functionality behind, in the provider context, + // but it's just a way for the "provider" block to create depedencies + // to other blocks. + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "i_depend_on": { + Type: cty.String, + Description: "Non-functional configuration attribute of the provider. This is meant to be used only to create depedencies of other resources to the provider block", + Optional: true, + }, + }, + }, }, ResourceTypes: map[string]providers.Schema{ - "simple_resource": simpleResource, + "simple_resource": withWriteOnlyAttribute(simpleResource), }, DataSources: map[string]providers.Schema{ "simple_resource": simpleResource, }, + EphemeralResources: map[string]providers.Schema{ + "simple_resource": simpleResource, + }, ServerCapabilities: providers.ServerCapabilities{ PlanDestroy: true, }, @@ -71,6 +100,10 @@ func (s simple) ValidateDataResourceConfig(_ context.Context, req providers.Vali return resp } +func (s simple) ValidateEphemeralConfig(context.Context, providers.ValidateEphemeralConfigRequest) (resp providers.ValidateEphemeralConfigResponse) { + return resp +} + func (s simple) MoveResourceState(_ context.Context, req providers.MoveResourceStateRequest) providers.MoveResourceStateResponse { var resp providers.MoveResourceStateResponse val, err := ctyjson.Unmarshal(req.SourceStateJSON, s.schema.ResourceTypes["simple_resource"].Block.ImpliedType()) @@ -119,6 +152,16 @@ func (s simple) PlanResourceChange(_ context.Context, req providers.PlanResource m["id"] = cty.UnknownVal(cty.String) } + // TODO ephemeral - remove this line after work will be done on write-only arguments. + // The problem now is that the value sent to ApplyResourceChange is always null as returned by the plan call. + // When the work on write-only arguments will be done, OpenTofu should send the actual value to + // the ApplyResourceChange too. + // To confirm that everything is ok, by removing this "waitIfRequested" call from here, theTestEphemeralWorkflowAndOutput + // should still work correctly without any warn logs in the test output + waitIfRequested(m) + // Simulate what the terraform-plugin-go should do. Nullify the write-only attributes. + m["value_wo"] = cty.NullVal(cty.String) + resp.PlannedState = cty.ObjectVal(m) return resp } @@ -134,6 +177,10 @@ func (s simple) ApplyResourceChange(_ context.Context, req providers.ApplyResour if !ok { m["id"] = cty.StringVal(time.Now().String()) } + waitIfRequested(m) + + // Simulate what the terraform-plugin-go should do. Nullify the write-only attributes. + m["value_wo"] = cty.NullVal(cty.String) resp.NewState = cty.ObjectVal(m) return resp @@ -151,6 +198,29 @@ func (s simple) ReadDataSource(_ context.Context, req providers.ReadDataSourceRe return resp } +func (s simple) OpenEphemeralResource(_ context.Context, request providers.OpenEphemeralResourceRequest) (resp providers.OpenEphemeralResourceResponse) { + m := request.Config.AsValueMap() + m["id"] = cty.StringVal("static-ephemeral-id") + if v, ok := m["value"]; ok && !v.IsNull() && strings.Contains(v.AsString(), "with-renew") { + t := time.Now().Add(200 * time.Millisecond) + resp.RenewAt = &t + } + resp.Result = cty.ObjectVal(m) + resp.Private = []byte("static private data") + return resp +} + +func (s simple) RenewEphemeralResource(_ context.Context, request providers.RenewEphemeralResourceRequest) (resp providers.RenewEphemeralResourceResponse) { + resp.Private = request.Private + t := time.Now().Add(200 * time.Millisecond) + resp.RenewAt = &t + return resp +} + +func (s simple) CloseEphemeralResource(_ context.Context, _ providers.CloseEphemeralResourceRequest) (resp providers.CloseEphemeralResourceResponse) { + return resp +} + func (s simple) GetFunctions(_ context.Context) providers.GetFunctionsResponse { panic("Not Implemented") } @@ -162,3 +232,12 @@ func (s simple) CallFunction(_ context.Context, r providers.CallFunctionRequest) func (s simple) Close(_ context.Context) error { return nil } + +func waitIfRequested(m map[string]cty.Value) { + // This is a special case that can be used together with ephemeral resources to be able to test the renewal process. + // When the "value" attribute of the resource is containing "with-renew" it will return later to allow + // the ephemeral resource to call renew at least once. Check also OpenEphemeralResource. + if v, ok := m["value_wo"]; ok && !v.IsNull() && strings.Contains(v.AsString(), "with-renew") { + <-time.After(time.Second) + } +} diff --git a/internal/providers/deferral.go b/internal/providers/deferral.go index b3c41157be..29af89332b 100644 --- a/internal/providers/deferral.go +++ b/internal/providers/deferral.go @@ -42,13 +42,13 @@ const ( // for how to skip the affected request so that other unaffected requests can // still be completed. func NewDeferralDiagnostic(reason DeferralReason) tfdiags.Diagnostic { - var summary, detail string + summary := DeferralReasonSummary(reason) + + var detail string switch reason { case DeferredBecauseResourceConfigUnknown: - summary = "Resource configuration is incomplete" detail = "The provider was unable to act on this resource configuration because it makes use of values from other resources that will not be known until after apply." case DeferredBecauseProviderConfigUnknown: - summary = "Provider configuration is incomplete" detail = "The provider was unable to work with this resource because the associated provider configuration makes use of values from other resources that will not be known until after apply." default: // This is the most general (and therefore least helpful) message, which @@ -61,7 +61,6 @@ func NewDeferralDiagnostic(reason DeferralReason) tfdiags.Diagnostic { // practice. If it becomes used in more providers in future then we can // hopefully devise a better message that describes what those providers // use it to mean.) - summary = "Operation cannot be completed yet" detail = "The provider reported that it is not able to perform the requested operation until more information is available." } @@ -79,6 +78,20 @@ func NewDeferralDiagnostic(reason DeferralReason) tfdiags.Diagnostic { }) } +// DeferralReasonSummary returns a more informative string representation of the given DeferralReason to be used +// it other places too. +// For more details, check the comments from NewDeferralDiagnostic. +func DeferralReasonSummary(reason DeferralReason) string { + switch reason { + case DeferredBecauseResourceConfigUnknown: + return "Resource configuration is incomplete" + case DeferredBecauseProviderConfigUnknown: + return "Provider configuration is incomplete" + default: + return "Operation cannot be completed yet" + } +} + // IsDeferralDiagnostic returns true if the given diagnostic was constructed // with NewDeferralDiagnostic, meaning that it describes a situation where a // provider reported that it is not yet able to complete an operation with the diff --git a/internal/providers/provider.go b/internal/providers/provider.go index 746dd56faa..d8a472fca0 100644 --- a/internal/providers/provider.go +++ b/internal/providers/provider.go @@ -7,6 +7,7 @@ package providers import ( "context" + "time" "github.com/zclconf/go-cty/cty" @@ -40,6 +41,10 @@ type Unconfigured interface { // configuration values. ValidateDataResourceConfig(context.Context, ValidateDataResourceConfigRequest) ValidateDataResourceConfigResponse + // ValidateEphemeralConfig allows the provider to validate the ephemeral resource + // configuration values. + ValidateEphemeralConfig(context.Context, ValidateEphemeralConfigRequest) ValidateEphemeralConfigResponse + // MoveResourceState requests that the given resource data be moved from one // type to another, potentially between providers as well. MoveResourceState(context.Context, MoveResourceStateRequest) MoveResourceStateResponse @@ -114,6 +119,32 @@ type Configured interface { // ReadDataSource returns the data source's current state. ReadDataSource(context.Context, ReadDataSourceRequest) ReadDataSourceResponse + // OpenEphemeralResource opens the provided ephemeral resource. + // This is meant to return the following: + // * the ephemeral information that will be used in other ephemeral contexts. + // The OpenEphemeralResourceResponse.Result is meant to be used all the time it's requested + // but this information will not be changed if the Renew will be called. + // Renew is meant to be supported only by a limited number of providers where the actual + // information from Result renewed by updating a remote state (eg: Vault/OpenBao) + // * internal private information that needs to be used for future Renew/Close calls. + // * a timestamp that will be used to determine if and when Renew call will be performed. + // * deferred information containing a reason returned by the provider. This will be used to + // determine if the resource needs to be deferred or not. + OpenEphemeralResource(context.Context, OpenEphemeralResourceRequest) OpenEphemeralResourceResponse + + // RenewEphemeralResource is renewing the information related to the OpenEphemeralResourceResponse.Result returned by + // the OpenEphemeralResource. + // The request is using the private information from the OpenEphemeralResourceResponse.Private + // to enable the provider to perform this action. + // The information returned in RenewEphemeralResourceResponse.Private needs to be used in any future call to + // Renew/Close. + RenewEphemeralResource(context.Context, RenewEphemeralResourceRequest) (resp RenewEphemeralResourceResponse) + + // CloseEphemeralResource closes the provided ephemeral resource. + // This requires the information from OpenEphemeralResourceResponse.Private or RenewEphemeralResourceResponse.Private + // to succeed. + CloseEphemeralResource(context.Context, CloseEphemeralResourceRequest) CloseEphemeralResourceResponse + // GetFunctions returns a full list of functions defined in this provider. It should be a super // set of the functions returned in GetProviderSchema() GetFunctions(context.Context) GetFunctionsResponse @@ -152,6 +183,9 @@ type GetProviderSchemaResponse struct { // Functions lists all functions supported by this provider. Functions map[string]FunctionSpec + + // EphemeralResources maps the ephemeral type name to that type's schema. + EphemeralResources map[string]Schema } // Schema pairs a provider or resource schema with that schema's version. @@ -264,6 +298,20 @@ type ValidateDataResourceConfigResponse struct { Diagnostics tfdiags.Diagnostics } +type ValidateEphemeralConfigRequest struct { + // TypeName is the name of the ephemeral resource type to validate. + TypeName string + + // Config is the configuration value to validate, which may contain unknown + // values. + Config cty.Value +} + +type ValidateEphemeralConfigResponse struct { + // Diagnostics contains any warnings or errors from the method call. + Diagnostics tfdiags.Diagnostics +} + type UpgradeResourceStateRequest struct { // TypeName is the name of the resource type being upgraded TypeName string @@ -547,6 +595,63 @@ type ReadDataSourceResponse struct { Diagnostics tfdiags.Diagnostics } +type OpenEphemeralResourceRequest struct { + // TypeName is the name of the ephemeral resource type to Open. + TypeName string + + // Config is the complete configuration for the requested ephemeral resource. + Config cty.Value +} + +type OpenEphemeralResourceResponse struct { + // Result will contain the ephemeral information returned by the ephemeral resource. + Result cty.Value + // Private is the provider information that needs to be used later on Renew/Close call. + Private []byte + // Deferred returns only a reason of why the provider is asking deferring the opening. + Deferred *EphemeralResourceDeferred + // RenewAt indicates if(!=nil) and when(<=time.Now()) the Renew call needs to be performed. + RenewAt *time.Time + + // Diagnostics contains any warnings or errors from the method call. + Diagnostics tfdiags.Diagnostics +} + +type EphemeralResourceDeferred struct { + DeferralReason DeferralReason +} + +type RenewEphemeralResourceRequest struct { + // TypeName is the name of the ephemeral resource to Renew. + TypeName string + + // Private should be the same with the one from the last call on Open/Renew call. + Private []byte +} + +type RenewEphemeralResourceResponse struct { + // Private needs to be used for the next call on Renew/Close + Private []byte + // RenewAt indicates if(!=nil) and when(<=time.Now()) the Renew call needs to be performed. + RenewAt *time.Time + + // Diagnostics contains any warnings or errors from the method call. + Diagnostics tfdiags.Diagnostics +} + +type CloseEphemeralResourceRequest struct { + // TypeName is the name of the ephemeral resource to Close. + TypeName string + + // Private should be the same with the one from the last call on Open/Renew call. + Private []byte +} + +type CloseEphemeralResourceResponse struct { + // Diagnostics contains any warnings or errors from the method call. + Diagnostics tfdiags.Diagnostics +} + type GetFunctionsResponse struct { Functions map[string]FunctionSpec diff --git a/internal/providers/schemas.go b/internal/providers/schemas.go index 8b9af95b02..59b36cacd9 100644 --- a/internal/providers/schemas.go +++ b/internal/providers/schemas.go @@ -25,6 +25,9 @@ func (ss ProviderSchema) SchemaForResourceType(mode addrs.ResourceMode, typeName case addrs.DataResourceMode: // Data resources don't have schema versions right now, since state is discarded for each refresh return ss.DataSources[typeName].Block, 0 + case addrs.EphemeralResourceMode: + // Ephemeral resources don't have schema versions right now, since state is discarded for each refresh + return ss.EphemeralResources[typeName].Block, 0 default: // Shouldn't happen, because the above cases are comprehensive. return nil, 0 diff --git a/internal/repl/session_test.go b/internal/repl/session_test.go index f6d257fc35..de2eba3259 100644 --- a/internal/repl/session_test.go +++ b/internal/repl/session_test.go @@ -85,7 +85,7 @@ func TestSession_basicState(t *testing.T) { { Input: "test_instance.bar.id", Error: true, - ErrorContains: `A managed resource "test_instance" "bar" has not been declared`, + ErrorContains: `There is no managed resource "test_instance" "bar" definition in the root module`, }, }, }) @@ -196,7 +196,7 @@ func TestSession_stateless(t *testing.T) { { Input: "test_instance.bar.id", Error: true, - ErrorContains: `resource "test_instance" "bar" has not been declared`, + ErrorContains: `resource "test_instance" "bar" definition`, }, }, }) diff --git a/internal/states/statefile/version3_upgrade.go b/internal/states/statefile/version3_upgrade.go index c038bcc116..8757a74460 100644 --- a/internal/states/statefile/version3_upgrade.go +++ b/internal/states/statefile/version3_upgrade.go @@ -98,6 +98,8 @@ func upgradeStateV3ToV4(old *stateV3) (*stateV4, error) { case addrs.DataResourceMode: modeStr = "data" default: + // This covers also addrs.EphemeralResourceMode. Should never happen, so this comment is just an indication + // that this part was checked during ephemeral resources implementation. return nil, fmt.Errorf("state contains resource %s with an unsupported resource mode %#v", resAddr, resAddr.Mode) } @@ -368,6 +370,10 @@ func upgradeInstanceObjectV3ToV4(_ *resourceStateV2, isOld *instanceStateV2, ins // parseLegacyResourceAddress parses the different identifier format used // state formats before version 4, like "instance.name.0". +// +// This function intentionally is not handling anything related to ephemeral resources since the ephemeral +// feature was introduced long after OpenTofu migrated to v4 state files. Therefore, v3 state files +// should never contain things related to ephemeral. func parseLegacyResourceAddress(s string) (addrs.ResourceInstance, error) { var ret addrs.ResourceInstance diff --git a/internal/states/statefile/version4.go b/internal/states/statefile/version4.go index 67dd7ff87e..3ab51bb874 100644 --- a/internal/states/statefile/version4.go +++ b/internal/states/statefile/version4.go @@ -72,6 +72,8 @@ func prepareStateV4(sV4 *stateV4) (*File, tfdiags.Diagnostics) { case "data": rAddr.Mode = addrs.DataResourceMode default: + // This covers also addrs.EphemeralResourceMode. Should never happen, so this comment is just an indication + // that this part was checked during ephemeral resources implementation. diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, "Invalid resource mode in state", @@ -426,6 +428,11 @@ func writeStateV4(file *File, w io.Writer, enc encryption.StateEncryption) tfdia mode = "managed" case addrs.DataResourceMode: mode = "data" + case addrs.EphemeralResourceMode: + // Ephemeral resources are the resources that are meant not to be written to the state file. + // Therefore, even though we can find those in the state (for evaluation reasons), we want to + // skip these from the state file. + continue default: diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, diff --git a/internal/tofu/context_apply2_test.go b/internal/tofu/context_apply2_test.go index 0a36bc5313..ccb7207f3f 100644 --- a/internal/tofu/context_apply2_test.go +++ b/internal/tofu/context_apply2_test.go @@ -5064,6 +5064,7 @@ output "test-child" { }{ "simpleModCall": { expectedWarn: tfdiags.Description{ + Address: "test_object.test", Summary: "Value derived from a deprecated source", Detail: "This value is derived from module.mod.test-child, which is deprecated with the following message:\n\nDon't use me", }, @@ -5104,6 +5105,7 @@ output "test-child" { }, "modForEach": { expectedWarn: tfdiags.Description{ + Address: "test_object.test", Summary: "Value derived from a deprecated source", Detail: "This value is derived from module.mod[\"a\"].test-child, which is deprecated with the following message:\n\nDon't use me", }, @@ -5288,8 +5290,8 @@ module "modfe" { t.Fatalf("Expected a warning, got: %v", diags.ErrWithWarnings()) } - if !diags[0].Description().Equal(test.expectedWarn) { - t.Fatalf("Unexpected warning: %v", diags.ErrWithWarnings()) + if got, want := diags[0].Description(), test.expectedWarn; !got.Equal(want) { + t.Fatalf("Unexpected warning. Want:\n%v\nGot:\n%v\n", want, got) } }) } @@ -5603,3 +5605,86 @@ check "http_check" { t.Fatal(diags.Err()) } } + +// TestContext2Apply_ephemeralResourcesLifecycleCheck is checking the hook calls +// and the state to be sure that the expected information is there. +func TestContext2Apply_ephemeralResourcesLifecycleCheck(t *testing.T) { + m := testModuleInline(t, map[string]string{ + `main.tf`: ` +ephemeral "test_ephemeral_resource" "a" { +} +`, + }) + + provider := testProvider("test") + provider.OpenEphemeralResourceResponse = &providers.OpenEphemeralResourceResponse{ + Result: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id val"), + "secret": cty.StringVal("val"), + }), + } + + ps := map[addrs.Provider]providers.Factory{ + addrs.NewDefaultProvider("test"): testProviderFuncFixed(provider), + } + + h := &testHook{} + apply := func(t *testing.T, m *configs.Config, prevState *states.State) (*states.State, tfdiags.Diagnostics) { + ctx := testContext2(t, &ContextOpts{ + Providers: ps, + Hooks: []Hook{h}, + }) + + plan, diags := ctx.Plan(context.Background(), m, prevState, &PlanOpts{ + Mode: plans.NormalMode, + }) + if diags.HasErrors() { + return nil, diags + } + + return ctx.Apply(context.Background(), plan, m) + } + + newState, diags := apply(t, m, states.NewState()) + if diags.HasErrors() { + t.Fatal(diags.Err()) + } + + addr := mustAbsResourceAddr("ephemeral.test_ephemeral_resource.a") + gotRes := newState.Resource(addr) + wantRes := &states.Resource{ + Addr: addr, + Instances: map[addrs.InstanceKey]*states.ResourceInstance{ + addrs.NoKey: { + Current: &states.ResourceInstanceObjectSrc{ + AttrsJSON: []byte(`{"id":"id val","secret":"val"}`), + Status: states.ObjectReady, + AttrSensitivePaths: []cty.PathValueMarks{}, + Dependencies: []addrs.ConfigResource{}, + }, + Deposed: map[states.DeposedKey]*states.ResourceInstanceObjectSrc{}, + }, + }, + ProviderConfig: mustProviderConfig(`provider["registry.opentofu.org/hashicorp/test"]`), + } + if diff := cmp.Diff(wantRes, gotRes); diff != "" { + t.Errorf("unexpected ephemeral resource content in the state:\n%s", diff) + } + + if got, want := len(h.Calls), 8; got != want { + t.Fatalf("want %d hook calls but got %d", want, got) + } + wantCalls := []*testHookCall{ + {Action: "PreOpen", InstanceID: addr.String()}, + {Action: "PostOpen", InstanceID: addr.String()}, + {Action: "PreClose", InstanceID: addr.String()}, + {Action: "PostClose", InstanceID: addr.String()}, + {Action: "PreOpen", InstanceID: addr.String()}, + {Action: "PostOpen", InstanceID: addr.String()}, + {Action: "PreClose", InstanceID: addr.String()}, + {Action: "PostClose", InstanceID: addr.String()}, + } + if diff := cmp.Diff(wantCalls, h.Calls); diff != "" { + t.Fatalf("unexpected hook calls:\n%s", diff) + } +} diff --git a/internal/tofu/context_input.go b/internal/tofu/context_input.go index a60d2476fe..64555a9593 100644 --- a/internal/tofu/context_input.go +++ b/internal/tofu/context_input.go @@ -72,7 +72,7 @@ func (c *Context) Input(ctx context.Context, config *configs.Config, mode InputM // We prompt for input only for provider configurations defined in // the root module. Provider configurations in other modules are a - // legacy thing we no longer recommend, and even if they weren't we + // legacy thing we no longer recommend, and even if they weren't, we // can't practically prompt for their inputs here because we've not // yet done "expansion" and so we don't know whether the modules are // using count or for_each. @@ -88,28 +88,9 @@ func (c *Context) Input(ctx context.Context, config *configs.Config, mode InputM // We also need to detect _implied_ provider configs from resources. // These won't have *configs.Provider objects, but they will still // exist in the map and we'll just treat them as empty below. - for _, rc := range config.Module.ManagedResources { - pa := rc.ProviderConfigAddr() - if pa.Alias != "" { - continue // alias configurations cannot be implied - } - if _, exists := pcs[pa.String()]; !exists { - pcs[pa.String()] = nil - pas[pa.String()] = pa - log.Printf("[TRACE] Context.Input: Provider %s implied by resource block at %s", pa, rc.DeclRange) - } - } - for _, rc := range config.Module.DataResources { - pa := rc.ProviderConfigAddr() - if pa.Alias != "" { - continue // alias configurations cannot be implied - } - if _, exists := pcs[pa.String()]; !exists { - pcs[pa.String()] = nil - pas[pa.String()] = pa - log.Printf("[TRACE] Context.Input: Provider %s implied by data block at %s", pa, rc.DeclRange) - } - } + collectResourcesImplicitProvider(config.Module.ManagedResources, pcs, pas) + collectResourcesImplicitProvider(config.Module.DataResources, pcs, pas) + collectResourcesImplicitProvider(config.Module.EphemeralResources, pcs, pas) for pk, pa := range pas { pc := pcs[pk] // will be nil if this is an implied config @@ -221,3 +202,21 @@ func schemaForInputSniffing(schema *hcl.BodySchema) *hcl.BodySchema { return ret } + +// collectResourcesImplicitProvider collects in "pas" all the provider addresses that are not having an explicit configuration +// existing in "pcs". In other words, it collects all the implicit providers from resources that are not explicitly configured. +// Later on, this information is used to determine what implicit providers need additional configuration and OpenTofu +// will ask for input on configuring those. +func collectResourcesImplicitProvider(resources map[string]*configs.Resource, pcs map[string]*configs.Provider, pas map[string]addrs.LocalProviderConfig) { + for _, rc := range resources { + pa := rc.ProviderConfigAddr() + if pa.Alias != "" { + continue // alias configurations cannot be implied + } + if _, exists := pcs[pa.String()]; !exists { + pcs[pa.String()] = nil + pas[pa.String()] = pa + log.Printf("[TRACE] Context.Input: Provider %s implied by %s block at %s", pa, addrs.ResourceModeBlockName(rc.Addr().Mode), rc.DeclRange) + } + } +} diff --git a/internal/tofu/context_input_test.go b/internal/tofu/context_input_test.go index cf7c566480..06320f317f 100644 --- a/internal/tofu/context_input_test.go +++ b/internal/tofu/context_input_test.go @@ -23,45 +23,90 @@ import ( func TestContext2Input_provider(t *testing.T) { m := testModule(t, "input-provider") - p := testProvider("aws") - p.GetProviderSchemaResponse = getProviderSchemaResponseFromProviderSchema(&ProviderSchema{ - Provider: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "foo": { - Type: cty.String, - Required: true, - Description: "something something", - }, + + providerCfgSchema := configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "foo": { + Type: cty.String, + Required: true, + Description: "something something", }, }, - ResourceTypes: map[string]*configschema.Block{ - "aws_instance": { - Attributes: map[string]*configschema.Attribute{ - "id": { - Type: cty.String, - Computed: true, - }, - }, + } + resourceCfgSchema := configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": { + Type: cty.String, + Computed: true, }, }, + } + // Create an aws provider with one resource + awsp := testProvider("aws") + awsp.GetProviderSchemaResponse = getProviderSchemaResponseFromProviderSchema(&ProviderSchema{ + Provider: &providerCfgSchema, + ResourceTypes: map[string]*configschema.Block{ + "aws_instance": &resourceCfgSchema, + }, }) + // Create a cloudflare provider with one data source + cfp := testProvider("cloudflare") + cfp.GetProviderSchemaResponse = getProviderSchemaResponseFromProviderSchema(&ProviderSchema{ + Provider: &providerCfgSchema, + DataSources: map[string]*configschema.Block{ + "cloudflare_account": &resourceCfgSchema, + }, + }) + cfp.ReadDataSourceResponse = &providers.ReadDataSourceResponse{ + State: cty.ObjectVal(map[string]cty.Value{"id": cty.StringVal("data content")}), + } + // Create an azure provider with one ephemeral resource + azp := testProvider("azurem") + azp.GetProviderSchemaResponse = getProviderSchemaResponseFromProviderSchema(&ProviderSchema{ + Provider: &providerCfgSchema, + EphemeralTypes: map[string]*configschema.Block{ + "azurerm_key_vault_secret": &resourceCfgSchema, + }, + }) + azp.OpenEphemeralResourceResponse = &providers.OpenEphemeralResourceResponse{Result: cty.ObjectVal(map[string]cty.Value{"id": cty.StringVal("ephemeral result")})} inp := &MockUIInput{ InputReturnMap: map[string]string{ - "provider.aws.foo": "bar", + "provider.aws.foo": "bar", + "provider.cloudflare.foo": "baz", + "provider.azurerm.foo": "qux", }, } ctx := testContext2(t, &ContextOpts{ Providers: map[addrs.Provider]providers.Factory{ - addrs.NewDefaultProvider("aws"): testProviderFuncFixed(p), + addrs.NewDefaultProvider("aws"): testProviderFuncFixed(awsp), + addrs.NewDefaultProvider("cloudflare"): testProviderFuncFixed(cfp), + addrs.NewDefaultProvider("azurerm"): testProviderFuncFixed(azp), }, UIInput: inp, }) - var actual interface{} - p.ConfigureProviderFn = func(req providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) { - actual = req.Config.GetAttr("foo").AsString() + var ( + actual = map[addrs.Provider]interface{}{} + mu sync.Mutex + ) + awsp.ConfigureProviderFn = func(req providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) { + mu.Lock() + defer mu.Unlock() + actual[addrs.NewDefaultProvider("aws")] = req.Config.GetAttr("foo").AsString() + return + } + cfp.ConfigureProviderFn = func(req providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) { + mu.Lock() + defer mu.Unlock() + actual[addrs.NewDefaultProvider("cloudflare")] = req.Config.GetAttr("foo").AsString() + return + } + azp.ConfigureProviderFn = func(req providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) { + mu.Lock() + defer mu.Unlock() + actual[addrs.NewDefaultProvider("azurerm")] = req.Config.GetAttr("foo").AsString() return } @@ -83,8 +128,13 @@ func TestContext2Input_provider(t *testing.T) { t.Fatalf("apply errors: %s", diags.Err()) } - if !reflect.DeepEqual(actual, "bar") { - t.Fatalf("wrong result\ngot: %#v\nwant: %#v", actual, "bar") + want := map[addrs.Provider]interface{}{ + addrs.NewDefaultProvider("aws"): "bar", + addrs.NewDefaultProvider("cloudflare"): "baz", + addrs.NewDefaultProvider("azurerm"): "qux", + } + if !reflect.DeepEqual(actual, want) { + t.Fatalf("wrong result\ngot: %#v\nwant: %#v", actual, want) } } @@ -186,67 +236,6 @@ func TestContext2Input_providerOnce(t *testing.T) { } } -func TestContext2Input_providerId(t *testing.T) { - input := new(MockUIInput) - - m := testModule(t, "input-provider") - - p := testProvider("aws") - p.GetProviderSchemaResponse = getProviderSchemaResponseFromProviderSchema(&ProviderSchema{ - Provider: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "foo": { - Type: cty.String, - Required: true, - Description: "something something", - }, - }, - }, - ResourceTypes: map[string]*configschema.Block{ - "aws_instance": { - Attributes: map[string]*configschema.Attribute{ - "id": { - Type: cty.String, - Computed: true, - }, - }, - }, - }, - }) - - ctx := testContext2(t, &ContextOpts{ - Providers: map[addrs.Provider]providers.Factory{ - addrs.NewDefaultProvider("aws"): testProviderFuncFixed(p), - }, - UIInput: input, - }) - - var actual interface{} - p.ConfigureProviderFn = func(req providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) { - actual = req.Config.GetAttr("foo").AsString() - return - } - - input.InputReturnMap = map[string]string{ - "provider.aws.foo": "bar", - } - - if diags := ctx.Input(context.Background(), m, InputModeStd); diags.HasErrors() { - t.Fatalf("input errors: %s", diags.Err()) - } - - plan, diags := ctx.Plan(context.Background(), m, states.NewState(), DefaultPlanOpts) - assertNoErrors(t, diags) - - if _, diags := ctx.Apply(context.Background(), plan, m); diags.HasErrors() { - t.Fatalf("apply errors: %s", diags.Err()) - } - - if !reflect.DeepEqual(actual, "bar") { - t.Fatalf("wrong result\ngot: %#v\nwant: %#v", actual, "bar") - } -} - func TestContext2Input_providerOnly(t *testing.T) { input := new(MockUIInput) diff --git a/internal/tofu/context_plan2_test.go b/internal/tofu/context_plan2_test.go index 4f4888cf95..11e0894705 100644 --- a/internal/tofu/context_plan2_test.go +++ b/internal/tofu/context_plan2_test.go @@ -22,7 +22,6 @@ import ( "github.com/opentofu/opentofu/internal/checks" "github.com/zclconf/go-cty/cty" - // "github.com/opentofu/opentofu/internal/configs" "github.com/opentofu/opentofu/internal/configs/configschema" "github.com/opentofu/opentofu/internal/lang/marks" "github.com/opentofu/opentofu/internal/plans" @@ -4168,7 +4167,7 @@ func TestContext2Plan_preconditionErrors(t *testing.T) { { "data.foo.bar", "Reference to undeclared resource", - `A data resource "foo" "bar" has not been declared in the root module`, + `There is no data resource "foo" "bar" definition in the root module.`, }, { "test_resource.b.value", @@ -8436,6 +8435,111 @@ func TestContext2Plan_removedModuleButModuleBlockStillExists(t *testing.T) { } } +// TestContext2Plan_ephemeralResourceDeferred is testing that an ephemeral resource gets deferred +// correctly: +// * gets deferred when a dependency is having planned changes, so OpenEphemeralResource is not called. +// * gets deferred when the response from OpenEphemeralResource is indicating so. +func TestContext2Plan_ephemeralResourceDeferred(t *testing.T) { + // Ephemeral resource is deferred by opentofu itself, before calling OpenEphemeralResource. This is + // due to pending changes in the ephemeral's dependencies. + t.Run("before open", func(t *testing.T) { + m := testModuleInline(t, map[string]string{ + "main.tf": ` + resource "test_object" "testres" { + } + ephemeral "test_object" "testeph" { + depends_on = [ + test_object.testres + ] + } + `, + }) + + state := states.BuildState(func(s *states.SyncState) {}) + + p := simpleMockProvider() + ctx := testContext2(t, &ContextOpts{ + Providers: map[addrs.Provider]providers.Factory{ + addrs.NewDefaultProvider("test"): testProviderFuncFixed(p), + }, + }) + hook := &testHook{} + ctx.hooks = append(ctx.hooks, hook) + + _, diags := ctx.Plan(context.Background(), m, state, &PlanOpts{ + Mode: plans.NormalMode, + }) + + if diags.HasErrors() { + t.Fatalf("unexpected errors: %s", diags.Err()) + } + + // last call should have been on the ephemeral defer + deferCall := hook.Calls[len(hook.Calls)-1] + if wantAction, wantInstID := "Deferred", "ephemeral_test_object.testeph"; deferCall.Action != wantAction && deferCall.InstanceID != wantInstID { + t.Fatalf("expected the last call to be a %q for %q. got action %q for %q", wantAction, wantInstID, deferCall.Action, deferCall.InstanceID) + } + }) + // Ephemeral is deferred because of the defer reason returned from OpenEphemeralResource. + t.Run("from open", func(t *testing.T) { + m := testModuleInline(t, map[string]string{ + "main.tf": ` + resource "test_object" "testres" { + test_string = "test value" + } + ephemeral "test_object" "testeph" { + depends_on = [ + test_object.testres + ] + } + `, + }) + + addr := mustAbsResourceAddr("test_object.testres") + state := states.BuildState(func(s *states.SyncState) { + s.SetResourceInstanceCurrent(addr.Instance(addrs.NoKey), &states.ResourceInstanceObjectSrc{ + AttrsJSON: []byte(`{"test_string": "test value"}`), + Status: states.ObjectReady, + }, mustProviderConfig(`provider["registry.opentofu.org/hashicorp/test"]`), addrs.NoKey) + }) + + p := simpleMockProvider() + p.OpenEphemeralResourceResponse = &providers.OpenEphemeralResourceResponse{ + Result: cty.Value{}, + Deferred: &providers.EphemeralResourceDeferred{DeferralReason: providers.DeferredBecauseResourceConfigUnknown}, + } + p.PlanResourceChangeFn = func(req providers.PlanResourceChangeRequest) (resp providers.PlanResourceChangeResponse) { + cfg := req.Config.AsValueMap() + resp.PlannedState = cty.ObjectVal(cfg) + return resp + } + ctx := testContext2(t, &ContextOpts{ + Providers: map[addrs.Provider]providers.Factory{ + addrs.NewDefaultProvider("test"): testProviderFuncFixed(p), + }, + }) + hook := &testHook{} + ctx.hooks = append(ctx.hooks, hook) + + _, diags := ctx.Plan(context.Background(), m, state, &PlanOpts{ + Mode: plans.NormalMode, + }) + + if diags.HasErrors() { + t.Fatalf("unexpected errors: %s", diags.Err()) + } + + if !p.OpenEphemeralResourceCalled { + t.Fatal("expected OpenEphemeralResource to be called but it was not") + } + // last call should have been on the ephemeral defer + deferCall := hook.Calls[len(hook.Calls)-1] + if wantAction, wantInstID := "Deferred", "ephemeral_test_object.testeph"; deferCall.Action != wantAction && deferCall.InstanceID != wantInstID { + t.Fatalf("expected the last call to be a %q for %q. got action %q for %q", wantAction, wantInstID, deferCall.Action, deferCall.InstanceID) + } + }) +} + func TestContext2Plan_importResourceWithSensitiveDataSource(t *testing.T) { addr := mustResourceInstanceAddr("test_object.b") m := testModuleInline(t, map[string]string{ @@ -8601,6 +8705,72 @@ func TestContext2Plan_insufficient_block(t *testing.T) { } } +// Ensure that running plan on a configuration with ephemeral resources, +// the generated plan contains the expected changes +func TestContext2Plan_ephemeralResourceChangesGenerated(t *testing.T) { + m := testModuleInline(t, map[string]string{ + "main.tf": ` +ephemeral "test_ephemeral_resource" "a" { +} +`, + }) + testProvider := testProvider("test") + testProvider.OpenEphemeralResourceResponse = &providers.OpenEphemeralResourceResponse{ + Result: cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id val"), + "secret": cty.StringVal("val"), + }), + } + + state := states.NewState() + + ctx := testContext2(t, &ContextOpts{ + Providers: map[addrs.Provider]providers.Factory{ + addrs.NewDefaultProvider("test"): testProviderFuncFixed(testProvider), + }, + }) + + plan, diags := ctx.Plan(context.Background(), m, state, DefaultPlanOpts) + if diags.HasErrors() { + t.Fatalf("unexpected plan error: %s", diags) + } + if plan.Changes == nil { + t.Fatalf("expected to have some changes but got none") + } + if got, want := len(plan.Changes.Resources), 1; got != want { + t.Fatalf("expected to have %d changes but got %d", want, got) + } + got := plan.Changes.Resources[0] + addr := mustResourceInstanceAddr("ephemeral.test_ephemeral_resource.a") + schema := testProvider.ProviderSchema().EphemeralTypes[addr.Resource.Resource.Type] + objTy := schema.ImpliedType() + priorVal := cty.NullVal(objTy) + beforeVal, err := plans.NewDynamicValue(priorVal, objTy) + if err != nil { + t.Fatalf("unexpected error creating before val: %s", err) + } + afterVal, err := plans.NewDynamicValue(cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("id val"), + "secret": cty.StringVal("val"), + }), objTy) + if err != nil { + t.Fatalf("unexpected error creating after val: %s", err) + } + want := &plans.ResourceInstanceChangeSrc{ + Addr: addr, + PrevRunAddr: addr, + ProviderAddr: mustProviderConfig(`provider["registry.opentofu.org/hashicorp/test"]`), + ChangeSrc: plans.ChangeSrc{ + Action: plans.Open, + Before: beforeVal, + After: afterVal, + }, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected diff in the ephemeral resource recorded change:\n%s", diff) + } +} + func mockProviderWithFeaturesBlock() *MockProvider { return &MockProvider{ GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ diff --git a/internal/tofu/context_plugins.go b/internal/tofu/context_plugins.go index 74b1112332..b06015b238 100644 --- a/internal/tofu/context_plugins.go +++ b/internal/tofu/context_plugins.go @@ -122,6 +122,17 @@ func (cp *contextPlugins) ProviderSchema(ctx context.Context, addr addrs.Provide } } + for t, d := range resp.EphemeralResources { + if err := d.Block.InternalValidate(); err != nil { + return resp, fmt.Errorf("provider %s has invalid schema for ephemeral resource type %q, which is a bug in the provider: %w", addr, t, err) + } + if d.Version < 0 { + // We're not using the version numbers here yet, but we'll check + // for validity anyway in case we start using them in future. + return resp, fmt.Errorf("provider %s has invalid negative schema version for ephemeral resource type %q, which is a bug in the provider", addr, t) + } + } + return resp, nil } diff --git a/internal/tofu/context_test.go b/internal/tofu/context_test.go index 35440d6e4d..16dbe7e629 100644 --- a/internal/tofu/context_test.go +++ b/internal/tofu/context_test.go @@ -754,6 +754,21 @@ func testProviderSchema(name string) *providers.GetProviderSchemaResponse { }, }, }, + EphemeralTypes: map[string]*configschema.Block{ + name + "_ephemeral_resource": { + Attributes: map[string]*configschema.Attribute{ + "id": { + Type: cty.String, + Computed: true, + }, + "secret": { + Type: cty.String, + Optional: true, + Computed: true, + }, + }, + }, + }, }) } diff --git a/internal/tofu/eval_variable.go b/internal/tofu/eval_variable.go index d5f0984c5e..e0cfa24f19 100644 --- a/internal/tofu/eval_variable.go +++ b/internal/tofu/eval_variable.go @@ -54,6 +54,25 @@ func prepareFinalInputVariableValue(addr addrs.AbsInputVariableInstance, raw *In } } + if marks.Contains(raw.Value, marks.Ephemeral) && !cfg.Ephemeral { + log.Printf("[TRACE] prepareFinalInputVariableValue: %q references an ephemeral value but not configured accordingly", addr) + // For child modules variables, this logic is unnecessary since those variables + // do always have a SourceRange defined. + // We generate subj this way because of the root module variables. In many cases, + // the SourceRange can be missing for root module variables. + subj := cfg.DeclRange + if raw.HasSourceRange() { + subj = raw.SourceRange.ToHCL() + } + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: `Variable does not allow ephemeral value`, + Detail: fmt.Sprintf("The value used for the variable %q is ephemeral, but it is not configured to allow one.", cfg.Name), + Subject: subj.Ptr(), + }) + return cty.UnknownVal(cfg.Type), diags + } + var sourceRange tfdiags.SourceRange var nonFileSource string if raw.HasSourceRange() { @@ -124,18 +143,23 @@ func prepareFinalInputVariableValue(addr addrs.AbsInputVariableInstance, raw *In "The given value is not suitable for %s declared at %s: %s.", addr, cfg.DeclRange.String(), err, ) - subject = sourceRange.ToHCL().Ptr() // In some workflows, the operator running tofu does not have access to the variables // themselves. They are for example stored in encrypted files that will be used by the CI toolset // and not by the operator directly. In such a case, the failing secret value should not be // displayed to the operator - if cfg.Sensitive { + subject = cfg.DeclRange.Ptr() + switch { + case cfg.Ephemeral: + detail = fmt.Sprintf( + "The given value is not suitable for %s, which is ephemeral: %s. Invalid value defined at %s.", + addr, err, sourceRange.ToHCL(), + ) + case cfg.Sensitive: detail = fmt.Sprintf( "The given value is not suitable for %s, which is sensitive: %s. Invalid value defined at %s.", addr, err, sourceRange.ToHCL(), ) - subject = cfg.DeclRange.Ptr() } } diff --git a/internal/tofu/eval_variable_test.go b/internal/tofu/eval_variable_test.go index 64b7264f10..cb8c0d96cb 100644 --- a/internal/tofu/eval_variable_test.go +++ b/internal/tofu/eval_variable_test.go @@ -76,6 +76,11 @@ func TestPrepareFinalInputVariableValue(t *testing.T) { nullable = false type = string } + variable "constrained_string_ephemeral_required" { + ephemeral = true + nullable = false + type = string + } variable "complex_type_with_nested_default_optional" { type = set(object({ name = string @@ -215,6 +220,18 @@ func TestPrepareFinalInputVariableValue(t *testing.T) { ) default = {} } + variable "simple_ephemeral_marked" { + type = string + ephemeral = true + } + + variable "complex_type_object_in_object" { + type = object({ + inner_obj = object({ + attr = string + }) + }) + } ` cfg := testModuleInline(t, map[string]string{ "main.tf": cfgSrc, @@ -839,6 +856,30 @@ func TestPrepareFinalInputVariableValue(t *testing.T) { cty.UnknownVal(cty.String), ``, }, + // ephemeral + { + "complex_type_object_in_object", + cty.ObjectVal(map[string]cty.Value{ + "inner_obj": cty.ObjectVal(map[string]cty.Value{ + "attr": cty.StringVal("inner attribute").Mark(marks.Ephemeral), + }), + }), + cty.UnknownVal(cty.Object(map[string]cty.Type{"inner_obj": cty.Object(map[string]cty.Type{"attr": cty.String})})), + `Variable does not allow ephemeral value: The value used for the variable "complex_type_object_in_object" is ephemeral, but it is not configured to allow one.`, + }, + { + "simple_ephemeral_marked", + cty.StringVal("raw value"), + cty.StringVal("raw value"), + ``, + }, + { + // ephemeral value given to a non-ephemeral variable + "constrained_string_nullable_required", + cty.StringVal("raw value").Mark(marks.Ephemeral), + cty.UnknownVal(cty.String), + `Variable does not allow ephemeral value: The value used for the variable "constrained_string_nullable_required" is ephemeral, but it is not configured to allow one.`, + }, } for _, test := range tests { @@ -1012,18 +1053,21 @@ func TestPrepareFinalInputVariableValue(t *testing.T) { t.Run("SensitiveVariable error message variants, with source variants", func(t *testing.T) { tests := []struct { + varName string SourceType ValueSourceType SourceRange tfdiags.SourceRange WantTypeErr string HideSubject bool }{ { + "constrained_string_sensitive_required", ValueFromUnknown, tfdiags.SourceRange{}, "Invalid value for input variable: Unsuitable value for var.constrained_string_sensitive_required set from outside of the configuration: string required, but have object.", false, }, { + "constrained_string_sensitive_required", ValueFromConfig, tfdiags.SourceRange{ Filename: "example.tfvars", @@ -1033,11 +1077,29 @@ func TestPrepareFinalInputVariableValue(t *testing.T) { `Invalid value for input variable: The given value is not suitable for var.constrained_string_sensitive_required, which is sensitive: string required, but have object. Invalid value defined at example.tfvars:1,1-1.`, true, }, + { + "constrained_string_ephemeral_required", + ValueFromUnknown, + tfdiags.SourceRange{}, + "Invalid value for input variable: Unsuitable value for var.constrained_string_ephemeral_required set from outside of the configuration: string required, but have object.", + false, + }, + { + "constrained_string_ephemeral_required", + ValueFromConfig, + tfdiags.SourceRange{ + Filename: "example.tfvars", + Start: tfdiags.SourcePos(hcl.InitialPos), + End: tfdiags.SourcePos(hcl.InitialPos), + }, + `Invalid value for input variable: The given value is not suitable for var.constrained_string_ephemeral_required, which is ephemeral: string required, but have object. Invalid value defined at example.tfvars:1,1-1.`, + true, + }, } for _, test := range tests { t.Run(fmt.Sprintf("%s %s", test.SourceType, test.SourceRange.StartString()), func(t *testing.T) { - varAddr := addrs.InputVariable{Name: "constrained_string_sensitive_required"}.Absolute(addrs.RootModuleInstance) + varAddr := addrs.InputVariable{Name: test.varName}.Absolute(addrs.RootModuleInstance) varCfg := variableConfigs[varAddr.Variable.Name] t.Run("type error", func(t *testing.T) { rawVal := &InputValue{ diff --git a/internal/tofu/evaluate.go b/internal/tofu/evaluate.go index 8a0bb76815..0d23668303 100644 --- a/internal/tofu/evaluate.go +++ b/internal/tofu/evaluate.go @@ -266,11 +266,15 @@ func (d *evaluationStateData) GetInputVariable(_ context.Context, addr addrs.Inp // being liberal in what it accepts because the subsequent plan walk has // more information available and so can be more conservative. if d.Operation == walkValidate { - // Ensure variable sensitivity is captured in the validate walk + // Ensure variable marks are captured in the validate walk + v := cty.UnknownVal(config.Type) if config.Sensitive { - return cty.UnknownVal(config.Type).Mark(marks.Sensitive), diags + v = v.Mark(marks.Sensitive) } - return cty.UnknownVal(config.Type), diags + if config.Ephemeral { + v = v.Mark(marks.Ephemeral) + } + return v, diags } moduleAddrStr := d.ModulePath.String() @@ -304,10 +308,13 @@ func (d *evaluationStateData) GetInputVariable(_ context.Context, addr addrs.Inp val = cty.UnknownVal(config.Type) } - // Mark if sensitive + // Mark the variable's value based on the configuration it's having if config.Sensitive { val = val.Mark(marks.Sensitive) } + if config.Ephemeral { + val = val.Mark(marks.Ephemeral) + } return val, diags } @@ -743,6 +750,23 @@ func (d *evaluationStateData) GetResource(ctx context.Context, addr addrs.Resour // We should only end up here during the validate walk, // since later walks should have at least partial states populated // for all resources in the configuration. + if schema.Ephemeral { + // If the block that it's evaluated is an ephemeral one, we want to mark + // the cty.DynamicVal as ephemeral to ensure that the ephemeral references + // check is working properly during walkValidate. + // For the sake of consistency, we could use "schema.ValueMarks(...)" instead. + // Though, since that method it also gathers sensitive marks from all the nesting + // layers, based on the size of the schema and the level of nested objects, + // that could add a pretty significant performance penalty for marking in the end + // only the root object with the ephemeral mark (only the root object, because the + // returned slice of cty.PathValueMarks will not be applicable to the attributes + // of cty.DynamicVal, since it is having none). + ephemeralMark := cty.PathValueMarks{ + Path: make(cty.Path, 0), + Marks: cty.NewValueMarks(marks.Ephemeral), + } + return cty.DynamicVal.MarkWithPaths([]cty.PathValueMarks{ephemeralMark}), diags + } return cty.DynamicVal, diags } } @@ -810,8 +834,13 @@ func (d *evaluationStateData) GetResource(ctx context.Context, addr addrs.Resour } afterMarks := change.AfterValMarks - if schema.ContainsSensitive() { - // Now that we know that the schema contains sensitive marks, + if schema.ContainsMarks() { + if schema.Ephemeral { + // Since we are preparing to mark the whole value as ephemeral, we want to remove any other + // possible downstream ephemeral marks to avoid having the same mark on multiple layers. + afterMarks = removeEphemeralMarks(afterMarks) + } + // Now that we know that the schema contains sensitive and/or ephemeral marks, // Combine those marks together to ensure that the value is marked correctly but not double marked schemaMarks := schema.ValueMarks(val, nil) afterMarks = combinePathValueMarks(afterMarks, schemaMarks) @@ -837,14 +866,18 @@ func (d *evaluationStateData) GetResource(ctx context.Context, addr addrs.Resour val := instanceObjectSrc.Value - if schema.ContainsSensitive() { - var marks []cty.PathValueMarks - // Now that we know that the schema contains sensitive marks, + if schema.ContainsMarks() { + var valMarks []cty.PathValueMarks + // Now that we know that the schema contains sensitive and/or ephemeral marks, // Combine those marks together to ensure that the value is marked correctly but not double marked - val, marks = val.UnmarkDeepWithPaths() + val, valMarks = val.UnmarkDeepWithPaths() schemaMarks := schema.ValueMarks(val, nil) - - combined := combinePathValueMarks(marks, schemaMarks) + if schema.Ephemeral { + // Since we are preparing to mark the whole value as ephemeral, we want to remove any other + // possible downstream ephemeral marks to avoid having the same mark on multiple layers. + valMarks = removeEphemeralMarks(valMarks) + } + combined := combinePathValueMarks(valMarks, schemaMarks) val = val.MarkWithPaths(combined) } instances[key] = val @@ -1015,6 +1048,7 @@ func (d *evaluationStateData) GetOutput(_ context.Context, addr addrs.OutputValu if output.Sensitive { val = val.Mark(marks.Sensitive) } + // TODO ephemeral - ensure that output is getting ephemeral marks correctly if config.Deprecated != "" { isRemote := false diff --git a/internal/tofu/evaluate_test.go b/internal/tofu/evaluate_test.go index 3b4ed8441e..0716b86cd4 100644 --- a/internal/tofu/evaluate_test.go +++ b/internal/tofu/evaluate_test.go @@ -543,6 +543,191 @@ func TestEvaluatorGetResource_changes(t *testing.T) { } } +func TestEvaluatorGetResource_Ephemeral(t *testing.T) { + rc := &configs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_resource", + Name: "foo", + Config: configs.SynthBody("", map[string]cty.Value{ + "secret_name": cty.StringVal("foo"), + }), + Provider: mustProviderConfig(`provider["registry.opentofu.org/hashicorp/test"]`).Provider, + } + ephemeralSchema := providers.Schema{ + Block: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "id": { + Type: cty.String, + Computed: true, + }, + "value": { + Type: cty.String, + Computed: true, + }, + }, + BlockTypes: map[string]*configschema.NestedBlock{ + "nesting_map": { + Block: configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "foo": { + Type: cty.String, + Optional: true, + // Sensitive is added here to ensure that the mark is kept after processing the ephemeral ones + Sensitive: true, + }, + }, + }, + Nesting: configschema.NestingSet, + }, + }, + }, + } + tests := map[string]struct { + changes *plans.ChangesSync + state *states.SyncState + want cty.Value + }{ + "no changes and no state": { + plans.NewChanges().SyncWrapper(), + states.NewState().SyncWrapper(), + cty.DynamicVal.Mark(marks.Ephemeral), + }, + "with state and planned changes": { + plans.BuildChanges(func(sync *plans.ChangesSync) { + sync.AppendResourceInstanceChange( + &plans.ResourceInstanceChangeSrc{ + Addr: rc.Addr().Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + PrevRunAddr: rc.Addr().Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + DeposedKey: states.NotDeposed, + ProviderAddr: addrs.AbsProviderConfig{ + Provider: rc.Provider, + Module: addrs.RootModule, + }, + ChangeSrc: plans.ChangeSrc{ + After: encodeDynamicValue(t, cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("foo"), + "value": cty.StringVal("tacos"), + "nesting_map": cty.SetVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.StringVal("test"), + }), + }), + })), + AfterValMarks: []cty.PathValueMarks{ + { + Path: cty.GetAttrPath("nesting_map").Index(cty.ObjectVal(map[string]cty.Value{"foo": cty.StringVal("test")})).GetAttr("foo"), + Marks: map[interface{}]struct{}{ + // added the ephemeral mark here to validate that it is removed and the + // sensitive one is added based on the schema + marks.Ephemeral: {}, + }, + }, + }, + }, + }, + ) + }).SyncWrapper(), + states.BuildState(func(state *states.SyncState) { + state.SetResourceInstanceCurrent( + rc.Addr().Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + Status: states.ObjectPlanned, + AttrsJSON: []byte(`{"id":"foo", "val":"tacos"}`), + }, + addrs.AbsProviderConfig{ + Provider: rc.Provider, + Module: addrs.RootModule, + }, + addrs.NoKey, + ) + }).SyncWrapper(), + cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("foo"), + "value": cty.StringVal("tacos"), + "nesting_map": cty.SetVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + // expected to have this attribute marked as sensitive but not as ephemeral + // since the ephemeral one is meant to be only at the root block level. + "foo": cty.StringVal("test").Mark(marks.Sensitive), + }), + }), + }).Mark(marks.Ephemeral), + }, + "with object ready state and no changes": { + plans.BuildChanges(func(sync *plans.ChangesSync) {}).SyncWrapper(), + states.BuildState(func(state *states.SyncState) { + state.SetResourceInstanceCurrent( + rc.Addr().Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + &states.ResourceInstanceObjectSrc{ + Status: states.ObjectReady, + AttrsJSON: []byte(`{"id":"foo", "value":"tacos", "nesting_map": [{"foo": "test"}]}`), + }, + addrs.AbsProviderConfig{ + Provider: rc.Provider, + Module: addrs.RootModule, + }, + addrs.NoKey, + ) + }).SyncWrapper(), + cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("foo"), + "value": cty.StringVal("tacos"), + "nesting_map": cty.SetVal([]cty.Value{ + cty.ObjectVal(map[string]cty.Value{ + // expected to have this attribute marked as sensitive but not as ephemeral + // since the ephemeral one is meant to be only at the root block level. + "foo": cty.StringVal("test").Mark(marks.Sensitive), + }), + }), + }).Mark(marks.Ephemeral), + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + // having these here for easier reference in the test body + state := tt.state + changes := tt.changes + want := tt.want + + evaluator := &Evaluator{ + Meta: &ContextMeta{ + Env: "foo", + }, + Changes: changes, + Config: &configs.Config{ + Module: &configs.Module{ + EphemeralResources: map[string]*configs.Resource{ + rc.Addr().String(): rc, + }, + }, + }, + State: state, + Plugins: schemaOnlyProvidersForTesting(map[addrs.Provider]providers.ProviderSchema{ + addrs.NewDefaultProvider("test"): { + EphemeralResources: map[string]providers.Schema{ + "test_resource": ephemeralSchema, + }, + }, + }, t), + } + data := &evaluationStateData{ + Evaluator: evaluator, + } + scope := evaluator.Scope(data, nil, nil, nil) + + got, diags := scope.Data.GetResource(t.Context(), rc.Addr(), tfdiags.SourceRange{}) + + if len(diags) != 0 { + t.Errorf("unexpected diagnostics %s", spew.Sdump(diags)) + } + + if !got.RawEquals(want) { + t.Errorf("wrong result:\ngot: %#v\nwant: %#v", got, want) + } + }) + } +} + func TestEvaluatorGetModule(t *testing.T) { // Create a new evaluator with an existing state stateSync := states.BuildState(func(ss *states.SyncState) { diff --git a/internal/tofu/evaluate_valid.go b/internal/tofu/evaluate_valid.go index 5fb3775892..3e47f4efcc 100644 --- a/internal/tofu/evaluate_valid.go +++ b/internal/tofu/evaluate_valid.go @@ -11,7 +11,6 @@ import ( "sort" "github.com/hashicorp/hcl/v2" - "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/configs" "github.com/opentofu/opentofu/internal/didyoumean" @@ -202,6 +201,8 @@ func (d *evaluationStateData) staticValidateResourceReference(ctx context.Contex modeAdjective = "managed" case addrs.DataResourceMode: modeAdjective = "data" + case addrs.EphemeralResourceMode: + modeAdjective = "ephemeral" default: // should never happen modeAdjective = "" @@ -209,21 +210,14 @@ func (d *evaluationStateData) staticValidateResourceReference(ctx context.Contex cfg := modCfg.Module.ResourceByAddr(addr) if cfg == nil { - var suggestion string - // A common mistake is omitting the data. prefix when trying to refer - // to a data resource, so we'll add a special hint for that. - if addr.Mode == addrs.ManagedResourceMode { - candidateAddr := addr // not a pointer, so this is a copy - candidateAddr.Mode = addrs.DataResourceMode - if candidateCfg := modCfg.Module.ResourceByAddr(candidateAddr); candidateCfg != nil { - suggestion = fmt.Sprintf("\n\nDid you mean the data resource %s?", candidateAddr) - } - } + // A common mistake is omitting the "data." (or "ephemeral.") prefix when trying to refer + // to a data (or ephemeral) resource, so we'll add a special hint for that. + suggestion := candidateSuggestion(addr, modCfg) diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: `Reference to undeclared resource`, - Detail: fmt.Sprintf(`A %s resource %q %q has not been declared in %s.%s`, modeAdjective, addr.Type, addr.Name, moduleConfigDisplayAddr(modCfg.Path), suggestion), + Detail: fmt.Sprintf(`There is no %s resource %q %q definition in %s.%s`, modeAdjective, addr.Type, addr.Name, moduleConfigDisplayAddr(modCfg.Path), suggestion), Subject: rng.ToHCL().Ptr(), }) return diags @@ -260,7 +254,7 @@ func (d *evaluationStateData) staticValidateResourceReference(ctx context.Contex diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: `Invalid resource type`, - Detail: fmt.Sprintf(`A %s resource type %q is not supported by provider %q.`, modeAdjective, addr.Type, providerFqn.String()), + Detail: fmt.Sprintf(`The %s resource type %q is not supported by provider %q.`, modeAdjective, addr.Type, providerFqn.String()), Subject: rng.ToHCL().Ptr(), }) return diags @@ -290,6 +284,39 @@ func (d *evaluationStateData) staticValidateResourceReference(ctx context.Contex return diags } +// candidateSuggestion is trying to return a close candidate by simply trying to find a different type of resource +// with the same type and name to suggest. We are trying to do so because a common mistake is omitting the "data." (or "ephemeral.") +// prefix when trying to refer to a data (or ephemeral) resource, so we'll add a special hint for that. +func candidateSuggestion(addr addrs.Resource, cfg *configs.Config) interface{} { + candidateAddr := addr // not a pointer, so this is a copy + tpl := "\n\nDid you mean the %s resource %s?" + filterOnOtherModes := func(targetModes []addrs.ResourceMode) *configs.Resource { + for _, candidateMode := range targetModes { + candidateAddr.Mode = candidateMode + if b := cfg.Module.ResourceByAddr(candidateAddr); b != nil { + return b + } + } + return nil + } + switch addr.Mode { + case addrs.ManagedResourceMode: + if candidateCfg := filterOnOtherModes([]addrs.ResourceMode{addrs.DataResourceMode, addrs.EphemeralResourceMode}); candidateCfg != nil { + return fmt.Sprintf(tpl, addrs.ResourceModeBlockName(candidateAddr.Mode), candidateAddr) + } + case addrs.DataResourceMode: + if candidateCfg := filterOnOtherModes([]addrs.ResourceMode{addrs.ManagedResourceMode, addrs.EphemeralResourceMode}); candidateCfg != nil { + return fmt.Sprintf(tpl, addrs.ResourceModeBlockName(candidateAddr.Mode), candidateAddr) + } + case addrs.EphemeralResourceMode: + if candidateCfg := filterOnOtherModes([]addrs.ResourceMode{addrs.ManagedResourceMode, addrs.DataResourceMode}); candidateCfg != nil { + return fmt.Sprintf(tpl, addrs.ResourceModeBlockName(candidateAddr.Mode), candidateAddr) + } + } + + return "" +} + func (d *evaluationStateData) staticValidateModuleCallReference(modCfg *configs.Config, addr addrs.ModuleCall, remain hcl.Traversal, rng tfdiags.SourceRange) tfdiags.Diagnostics { var diags tfdiags.Diagnostics diff --git a/internal/tofu/evaluate_valid_test.go b/internal/tofu/evaluate_valid_test.go index 06047e9f5f..251e2cae0b 100644 --- a/internal/tofu/evaluate_valid_test.go +++ b/internal/tofu/evaluate_valid_test.go @@ -38,11 +38,11 @@ func TestStaticValidateReferences(t *testing.T) { }, { Ref: "aws_instance.nonexist", - WantErr: `Reference to undeclared resource: A managed resource "aws_instance" "nonexist" has not been declared in the root module.`, + WantErr: `Reference to undeclared resource: There is no managed resource "aws_instance" "nonexist" definition in the root module.`, }, { Ref: "beep.boop", - WantErr: `Reference to undeclared resource: A managed resource "beep" "boop" has not been declared in the root module. + WantErr: `Reference to undeclared resource: There is no managed resource "beep" "boop" definition in the root module. Did you mean the data resource data.beep.boop?`, }, @@ -70,7 +70,7 @@ For example, to correlate with indices of a referring resource, use: }, { Ref: "boop_whatever.nope", - WantErr: `Invalid resource type: A managed resource type "boop_whatever" is not supported by provider "registry.opentofu.org/foobar/beep".`, + WantErr: `Invalid resource type: The managed resource type "boop_whatever" is not supported by provider "registry.opentofu.org/foobar/beep".`, }, { Ref: "data.boop_data.boop_nested", @@ -81,6 +81,24 @@ For example, to correlate with indices of a referring resource, use: WantErr: ``, Src: addrs.Check{Name: "foo"}, }, + { + Ref: "foo.bar", + WantErr: `Reference to undeclared resource: There is no managed resource "foo" "bar" definition in the root module. + +Did you mean the ephemeral resource ephemeral.foo.bar?`, + }, + { + Ref: "data.foo.bar", + WantErr: `Reference to undeclared resource: There is no data resource "foo" "bar" definition in the root module. + +Did you mean the ephemeral resource ephemeral.foo.bar?`, + }, + { + Ref: "ephemeral.beep.boop", + WantErr: `Reference to undeclared resource: There is no ephemeral resource "beep" "boop" definition in the root module. + +Did you mean the data resource data.beep.boop?`, + }, } cfg := testModule(t, "static-validate-refs") diff --git a/internal/tofu/graph_builder_apply.go b/internal/tofu/graph_builder_apply.go index 1e9e44eea1..d3a5cd6331 100644 --- a/internal/tofu/graph_builder_apply.go +++ b/internal/tofu/graph_builder_apply.go @@ -199,6 +199,9 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer { // Target &TargetingTransformer{Targets: b.Targets, Excludes: b.Excludes}, + // Detect the ephemeral expandable resources and create nodes to close them + &CloseableResourceTransformer{}, + // Close opened plugin connections &CloseProviderTransformer{}, diff --git a/internal/tofu/graph_builder_plan.go b/internal/tofu/graph_builder_plan.go index e2ff1cf3f3..96ee3f748a 100644 --- a/internal/tofu/graph_builder_plan.go +++ b/internal/tofu/graph_builder_plan.go @@ -131,8 +131,19 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer { Concrete: b.ConcreteResource, Config: b.Config, - // Resources are not added from the config on destroy. - skip: b.Operation == walkPlanDestroy, + // Instead of just skipping the ConfigTransformer altogether during walkPlanDestroy, + // we want to add only the ephemeral resources. + // This is needed to be able later to use the changes generated to create + // actual applyable instance nodes to have the ephemeral information fetched + // for the nodes that depend on it (ie: configuring a "provider" block with ephemeral values) + ModeFilter: func(mode addrs.ResourceMode) bool { + if b.Operation != walkPlanDestroy { + // Allow all the resource types on the operations that are not walkPlanDestroy + return false + } + // For the walkPlanDestroy, allow only ephemeral resources + return mode != addrs.EphemeralResourceMode + }, importTargets: b.ImportTargets, @@ -226,9 +237,9 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer { &AttachDependenciesTransformer{}, - // Make sure data sources are aware of any depends_on from the - // configuration - &attachDataResourceDependsOnTransformer{}, + // Make sure data sources and ephemeral resources are aware of any + // depends_on from the configuration + &attachResourceDependsOnTransformer{}, // DestroyEdgeTransformer is only required during a plan so that the // TargetingTransformer can determine which nodes to keep in the graph. @@ -247,6 +258,12 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer { // node due to dependency edges, to avoid graph cycles during apply. &ForcedCBDTransformer{}, + // Detect the ephemeral plannable resources and create nodes to close them + &CloseableResourceTransformer{ + // Closeable nodes are not needed to be added during validate. + skip: b.Operation == walkValidate, + }, + // Close opened plugin connections &CloseProviderTransformer{}, diff --git a/internal/tofu/graph_builder_plan_test.go b/internal/tofu/graph_builder_plan_test.go index a6a1b720ab..1e93494a60 100644 --- a/internal/tofu/graph_builder_plan_test.go +++ b/internal/tofu/graph_builder_plan_test.go @@ -10,6 +10,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/opentofu/opentofu/internal/configs" + "github.com/opentofu/opentofu/internal/states" "github.com/zclconf/go-cty/cty" "github.com/opentofu/opentofu/internal/addrs" @@ -247,6 +249,69 @@ func TestPlanGraphBuilder_forEach(t *testing.T) { } } +// TestPlanGraphBuilder_ephemeralResourceDestroy contains some wierd and theoretically impossible setup steps, but it's done +// this way to verify that some checks are in place along the way. Check the inline comments for more details. +func TestPlanGraphBuilder_ephemeralResourceDestroy(t *testing.T) { + awsProvider := mockProviderWithResourceTypeSchema("aws_secretmanager_secret", simpleTestSchema()) + b := &PlanGraphBuilder{ + Config: &configs.Config{Module: &configs.Module{}}, + Operation: walkPlanDestroy, + Plugins: newContextPlugins(map[addrs.Provider]providers.Factory{ + addrs.NewDefaultProvider("aws"): providers.FactoryFixed(awsProvider), + }, nil), + State: &states.State{ + Modules: map[string]*states.Module{ + "": { + Resources: map[string]*states.Resource{ + // This is the wierd/stupid setup: the state is NEVER meant to contain an ephemeral resource. + // This setup is done this way only to be sure that the code path for creating NodePlanDestroyableResourceInstance + // is working well and that the node resulted from that returns an error on v.Execute(...) + "ephemeral.aws_secretmanager_secret.test": { + Addr: mustAbsResourceAddr("ephemeral.aws_secretmanager_secret.test"), + Instances: map[addrs.InstanceKey]*states.ResourceInstance{ + addrs.NoKey: { + Current: &states.ResourceInstanceObjectSrc{}, + }, + }, + }, + }, + }, + }, + }, + } + + g, err := b.Build(t.Context(), addrs.RootModuleInstance) + if err != nil { + t.Fatalf("err: %s", err) + } + + t.Logf("Graph: %s", g.String()) + var found *NodePlanDestroyableResourceInstance + for _, vertex := range g.Vertices() { + if v, ok := vertex.(*NodePlanDestroyableResourceInstance); ok { + if found == nil { + found = v + continue // not break on purpose to check if there are other NodePlanDestroyableResourceInstance in the graph + } + t.Fatal("found more than 1 NodePlanDestroyableResourceInstance in the graph") + } + } + if found == nil { + t.Fatal("expected to find one NodePlanDestroyableResourceInstance in graph") + } + + // Let's see how NodePlanDestroyableResourceInstance.Execute is behaving when it's for an ephemeral resource + evalCtx := &MockEvalContext{ + ProviderProvider: testProvider("aws"), + } + diags := found.Execute(t.Context(), evalCtx, walkPlanDestroy) + got := diags.Err().Error() + want := `An ephemeral resource planned for destroy: A destroy operation has been planned for the ephemeral resource "ephemeral.aws_secretmanager_secret.test". This is an OpenTofu error. Please report this.` + if got != want { + t.Fatalf("unexpected error returned.\ngot: %s\nwant:%s", got, want) + } +} + const testPlanGraphBuilderStr = ` aws_instance.web (expand) aws_security_group.firewall (expand) diff --git a/internal/tofu/hook.go b/internal/tofu/hook.go index 7a84448d12..c5ad809151 100644 --- a/internal/tofu/hook.go +++ b/internal/tofu/hook.go @@ -96,6 +96,22 @@ type Hook interface { PreApplyForget(addr addrs.AbsResourceInstance) (HookAction, error) PostApplyForget(addr addrs.AbsResourceInstance) (HookAction, error) + // Deferred is called when a resource is deferred from the plan phase due to + // a specific given reason. + Deferred(addr addrs.AbsResourceInstance, reason string) (HookAction, error) + + // PreOpen and PostOpen are called before and after the request to a provider + // to open an ephemeral resource. + PreOpen(addr addrs.AbsResourceInstance) (HookAction, error) + PostOpen(addr addrs.AbsResourceInstance, err error) (HookAction, error) + // PreRenew and PostRenew are called before and after the request to a provider + // to renew an ephemeral resource. + PreRenew(addr addrs.AbsResourceInstance) (HookAction, error) + PostRenew(addr addrs.AbsResourceInstance, err error) (HookAction, error) + // PreClose and PostClose are called before and after the request to a provider + // to close an ephemeral resource. + PreClose(addr addrs.AbsResourceInstance) (HookAction, error) + PostClose(addr addrs.AbsResourceInstance, err error) (HookAction, error) // Stopping is called if an external signal requests that OpenTofu // gracefully abort an operation in progress. // @@ -200,6 +216,34 @@ func (h *NilHook) PostApplyForget(_ addrs.AbsResourceInstance) (HookAction, erro return HookActionContinue, nil } +func (h *NilHook) Deferred(_ addrs.AbsResourceInstance, _ string) (HookAction, error) { + return HookActionContinue, nil +} + +func (h *NilHook) PreOpen(_ addrs.AbsResourceInstance) (HookAction, error) { + return HookActionContinue, nil +} + +func (h *NilHook) PostOpen(_ addrs.AbsResourceInstance, _ error) (HookAction, error) { + return HookActionContinue, nil +} + +func (h *NilHook) PreRenew(_ addrs.AbsResourceInstance) (HookAction, error) { + return HookActionContinue, nil +} + +func (h *NilHook) PostRenew(_ addrs.AbsResourceInstance, _ error) (HookAction, error) { + return HookActionContinue, nil +} + +func (h *NilHook) PreClose(_ addrs.AbsResourceInstance) (HookAction, error) { + return HookActionContinue, nil +} + +func (h *NilHook) PostClose(_ addrs.AbsResourceInstance, _ error) (HookAction, error) { + return HookActionContinue, nil +} + func (*NilHook) Stopping() { // Does nothing at all by default } diff --git a/internal/tofu/hook_mock.go b/internal/tofu/hook_mock.go index 0d21b6f33c..76f011e4a9 100644 --- a/internal/tofu/hook_mock.go +++ b/internal/tofu/hook_mock.go @@ -141,6 +141,44 @@ type MockHook struct { PostApplyForgetReturn HookAction PostApplyForgetError error + DeferredCalled bool + DeferredReturn HookAction + DeferredError error + + PreOpenCalled bool + PreOpenAddr addrs.AbsResourceInstance + PreOpenReturn HookAction + PreOpenError error + + PostOpenCalled bool + PostOpenAddr addrs.AbsResourceInstance + PostOpenError error + PostOpenReturn HookAction + PostOpenReturnError error + + PreRenewCalled bool + PreRenewAddr addrs.AbsResourceInstance + PreRenewReturn HookAction + PreRenewError error + + PostRenewCalled bool + PostRenewAddr addrs.AbsResourceInstance + PostRenewError error + PostRenewReturn HookAction + PostRenewReturnError error + + PreCloseCalled bool + PreCloseAddr addrs.AbsResourceInstance + PreCloseAction plans.Action + PreCloseReturn HookAction + PreCloseError error + + PostCloseCalled bool + PostCloseAddr addrs.AbsResourceInstance + PostCloseError error + PostCloseReturn HookAction + PostCloseReturnError error + StoppingCalled bool PostStateUpdateCalled bool @@ -351,6 +389,74 @@ func (h *MockHook) PostApplyForget(_ addrs.AbsResourceInstance) (HookAction, err return h.PostApplyForgetReturn, h.PostApplyForgetError } +func (h *MockHook) Deferred(_ addrs.AbsResourceInstance, _ string) (HookAction, error) { + h.Lock() + defer h.Unlock() + + h.DeferredCalled = true + return h.DeferredReturn, h.DeferredError +} + +func (h *MockHook) PreOpen(addr addrs.AbsResourceInstance) (HookAction, error) { + h.Lock() + defer h.Unlock() + + h.PreOpenCalled = true + h.PreOpenAddr = addr + return h.PreOpenReturn, h.PreOpenError +} + +func (h *MockHook) PostOpen(addr addrs.AbsResourceInstance, err error) (HookAction, error) { + h.Lock() + defer h.Unlock() + + h.PostOpenCalled = true + h.PostOpenAddr = addr + h.PostOpenError = err + + return h.PostOpenReturn, h.PostOpenReturnError +} + +func (h *MockHook) PreRenew(addr addrs.AbsResourceInstance) (HookAction, error) { + h.Lock() + defer h.Unlock() + + h.PreRenewCalled = true + h.PreRenewAddr = addr + return h.PreRenewReturn, h.PreRenewError +} + +func (h *MockHook) PostRenew(addr addrs.AbsResourceInstance, err error) (HookAction, error) { + h.Lock() + defer h.Unlock() + + h.PostRenewCalled = true + h.PostRenewAddr = addr + h.PostRenewError = err + + return h.PostRenewReturn, h.PostRenewReturnError +} + +func (h *MockHook) PreClose(addr addrs.AbsResourceInstance) (HookAction, error) { + h.Lock() + defer h.Unlock() + + h.PreCloseCalled = true + h.PreCloseAddr = addr + return h.PreCloseReturn, h.PreCloseError +} + +func (h *MockHook) PostClose(addr addrs.AbsResourceInstance, err error) (HookAction, error) { + h.Lock() + defer h.Unlock() + + h.PostCloseCalled = true + h.PostCloseAddr = addr + h.PostCloseError = err + + return h.PostCloseReturn, h.PostCloseReturnError +} + func (h *MockHook) Stopping() { h.Lock() defer h.Unlock() diff --git a/internal/tofu/hook_stop.go b/internal/tofu/hook_stop.go index 4e86e014d9..9ab5efa840 100644 --- a/internal/tofu/hook_stop.go +++ b/internal/tofu/hook_stop.go @@ -100,6 +100,34 @@ func (h *stopHook) PostApplyForget(_ addrs.AbsResourceInstance) (HookAction, err return h.hook() } +func (h *stopHook) Deferred(_ addrs.AbsResourceInstance, _ string) (HookAction, error) { + return h.hook() +} + +func (h *stopHook) PreOpen(_ addrs.AbsResourceInstance) (HookAction, error) { + return h.hook() +} + +func (h *stopHook) PostOpen(_ addrs.AbsResourceInstance, _ error) (HookAction, error) { + return h.hook() +} + +func (h *stopHook) PreRenew(_ addrs.AbsResourceInstance) (HookAction, error) { + return h.hook() +} + +func (h *stopHook) PostRenew(_ addrs.AbsResourceInstance, _ error) (HookAction, error) { + return h.hook() +} + +func (h *stopHook) PreClose(_ addrs.AbsResourceInstance) (HookAction, error) { + return h.hook() +} + +func (h *stopHook) PostClose(_ addrs.AbsResourceInstance, _ error) (HookAction, error) { + return h.hook() +} + func (h *stopHook) Stopping() {} func (h *stopHook) PostStateUpdate(new *states.State) (HookAction, error) { diff --git a/internal/tofu/hook_test.go b/internal/tofu/hook_test.go index ee8e827a0d..afb07ef904 100644 --- a/internal/tofu/hook_test.go +++ b/internal/tofu/hook_test.go @@ -171,6 +171,55 @@ func (h *testHook) PostApplyForget(addr addrs.AbsResourceInstance) (HookAction, return HookActionContinue, nil } +func (h *testHook) Deferred(addr addrs.AbsResourceInstance, reason string) (HookAction, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.Calls = append(h.Calls, &testHookCall{"Deferred", addr.String()}) + return HookActionContinue, nil +} + +func (h *testHook) PreOpen(addr addrs.AbsResourceInstance) (HookAction, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.Calls = append(h.Calls, &testHookCall{"PreOpen", addr.String()}) + return HookActionContinue, nil +} + +func (h *testHook) PostOpen(addr addrs.AbsResourceInstance, _ error) (HookAction, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.Calls = append(h.Calls, &testHookCall{"PostOpen", addr.String()}) + return HookActionContinue, nil +} + +func (h *testHook) PreRenew(addr addrs.AbsResourceInstance) (HookAction, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.Calls = append(h.Calls, &testHookCall{"PreRenew", addr.String()}) + return HookActionContinue, nil +} + +func (h *testHook) PostRenew(addr addrs.AbsResourceInstance, _ error) (HookAction, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.Calls = append(h.Calls, &testHookCall{"PostRenew", addr.String()}) + return HookActionContinue, nil +} + +func (h *testHook) PreClose(addr addrs.AbsResourceInstance) (HookAction, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.Calls = append(h.Calls, &testHookCall{"PreClose", addr.String()}) + return HookActionContinue, nil +} + +func (h *testHook) PostClose(addr addrs.AbsResourceInstance, _ error) (HookAction, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.Calls = append(h.Calls, &testHookCall{"PostClose", addr.String()}) + return HookActionContinue, nil +} + func (h *testHook) Stopping() { h.mu.Lock() defer h.mu.Unlock() diff --git a/internal/tofu/marks.go b/internal/tofu/marks.go index 799dd84370..da99fe5a04 100644 --- a/internal/tofu/marks.go +++ b/internal/tofu/marks.go @@ -128,3 +128,17 @@ func combinePathValueMarks(marks []cty.PathValueMarks, other []cty.PathValueMark return combined } + +// removeEphemeralMarks is meant to remove the marks.Ephemeral from any cty.PathValueMarks. +// This is needed to remove the aforementioned mark from the attributes of a value +// before marking the whole value with marks.Ephemeral. +func removeEphemeralMarks(ms []cty.PathValueMarks) []cty.PathValueMarks { + res := make([]cty.PathValueMarks, len(ms)) + for i, mark := range ms { + // Since we are preparing to mark the whole value as ephemeral, we want to remove any other + // possible downstream ephemeral marks to avoid having the same mark on multiple layers. + delete(mark.Marks, marks.Ephemeral) + res[i] = mark + } + return res +} diff --git a/internal/tofu/node_output.go b/internal/tofu/node_output.go index 14fbd31f73..6575993b58 100644 --- a/internal/tofu/node_output.go +++ b/internal/tofu/node_output.go @@ -371,6 +371,11 @@ func (n *NodeApplyableOutput) Execute(ctx context.Context, evalCtx EvalContext, // depends_on expressions here too diags = diags.Append(validateDependsOn(ctx, evalCtx, n.Config.DependsOn)) + // Before checking for the sensitivity, we want to check for ephemerality, since it's + // a more restrictive mark. + if ephDiags := n.validateEphemerality(val); ephDiags.HasErrors() { + return diags.Append(ephDiags) + } // For root module outputs in particular, an output value must be // statically declared as sensitive in order to dynamically return // a sensitive result, to help avoid accidental exposure in the state @@ -387,6 +392,14 @@ If you do intend to export this data, annotate the output value as sensitive by Subject: n.Config.DeclRange.Ptr(), }) } + if n.Config.Ephemeral { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid output configuration", + Detail: "Root modules are not allowed to have outputs defined as ephemeral", + Subject: n.Config.DeclRange.Ptr(), + }) + } } } @@ -417,6 +430,11 @@ If you do intend to export this data, annotate the output value as sensitive by } return diags } + + if ephDiags := n.validateEphemerality(val); ephDiags.HasErrors() { + return diags.Append(ephDiags) + } + n.setValue(state, changes, val) // If we were able to evaluate a new value, we can update that in the @@ -429,6 +447,26 @@ If you do intend to export this data, annotate the output value as sensitive by return diags } +func (n *NodeApplyableOutput) validateEphemerality(val cty.Value) (diags tfdiags.Diagnostics) { + // We don't want to check when the value is unknown and is not marked. + // If the value is unknown due to the referenced values and inherited the marks from those, + // we do want to validate though. + if !val.IsWhollyKnown() && !val.IsMarked() { + return diags + } + ephemeralMarked := marks.Contains(val, marks.Ephemeral) + // We don't allow ephemeral values in outputs that are not marked as such. + if !n.Config.Ephemeral && ephemeralMarked { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Output does not allow ephemeral value", + Detail: "The value that was generated for the output is ephemeral, but it is not configured to allow one.", + Subject: n.Config.UsageRange().Ptr(), + }) + } + return diags +} + // dag.GraphNodeDotter impl. func (n *NodeApplyableOutput) DotNode(name string, opts *dag.DotOpts) *dag.DotNode { return &dag.DotNode{ diff --git a/internal/tofu/node_output_test.go b/internal/tofu/node_output_test.go index 430cf2a80d..38508cd7d0 100644 --- a/internal/tofu/node_output_test.go +++ b/internal/tofu/node_output_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/hashicorp/hcl/v2" + "github.com/opentofu/opentofu/internal/plans" "github.com/zclconf/go-cty/cty" "github.com/opentofu/opentofu/internal/addrs" @@ -122,6 +123,104 @@ func TestNodeApplyableOutputExecute_sensitiveValueNotOutput(t *testing.T) { } } +func TestNodeApplyableOutputExecute_ephemerality(t *testing.T) { + t.Run("output not marked as ephemeral but got ephemeral value from evaluation", func(t *testing.T) { + evalCtx := new(MockEvalContext) + evalCtx.StateState = states.NewState().SyncWrapper() + evalCtx.ChecksState = checks.NewState(nil) + + config := &configs.Output{Name: "map-output"} + addr := addrs.OutputValue{Name: config.Name}.Absolute(addrs.RootModuleInstance) + node := &NodeApplyableOutput{Config: config, Addr: addr} + val := cty.MapVal(map[string]cty.Value{ + "a": cty.StringVal("b").Mark(marks.Ephemeral), + }) + evalCtx.EvaluateExprResult = val + + diags := node.Execute(t.Context(), evalCtx, walkApply) + if !diags.HasErrors() { + t.Fatal("expected execute error, but there was none") + } + if got, want := diags.Err().Error(), "Output does not allow ephemeral value: The value that was generated for the output is ephemeral, but it is not configured to allow one"; !strings.Contains(got, want) { + t.Errorf("expected error to include %q, but was: %s", want, got) + } + }) + t.Run("output marked as ephemeral in root module", func(t *testing.T) { + evalCtx := new(MockEvalContext) + evalCtx.StateState = states.NewState().SyncWrapper() + evalCtx.ChecksState = checks.NewState(nil) + + config := &configs.Output{Name: "map-output", Ephemeral: true} + addr := addrs.OutputValue{Name: config.Name}.Absolute(addrs.RootModuleInstance) + node := &NodeApplyableOutput{Config: config, Addr: addr} + val := cty.MapVal(map[string]cty.Value{ + "a": cty.StringVal("b").Mark(marks.Ephemeral), + }) + evalCtx.EvaluateExprResult = val + + diags := node.Execute(t.Context(), evalCtx, walkApply) + if !diags.HasErrors() { + t.Fatal("expected execute error, but there was none") + } + if got, want := diags.Err().Error(), "Invalid output configuration: Root modules are not allowed to have outputs defined as ephemeral"; !strings.Contains(got, want) { + t.Errorf("expected error to include %q, but was: %s", want, got) + } + }) + t.Run("output not marked as ephemeral but the registered change is ephemeral", func(t *testing.T) { + evalCtx := new(MockEvalContext) + evalCtx.StateState = states.NewState().SyncWrapper() + evalCtx.ChecksState = checks.NewState(nil) + + config := &configs.Output{Name: "map-output"} + addr := addrs.OutputValue{Name: config.Name}.Absolute(addrs.RootModuleInstance) + + change := &plans.OutputChange{ + Addr: addr, + Change: plans.Change{ + Action: plans.Create, + After: cty.StringVal("new value"), + }, + } + changeSrc, err := change.Encode() + if err != nil { + t.Fatalf("failed to encode the output change: %s", err) + } + changeSrc.AfterValMarks = []cty.PathValueMarks{{Marks: map[interface{}]struct{}{marks.Ephemeral: {}}}} + node := &NodeApplyableOutput{Config: config, Addr: addr, Change: changeSrc} + + diags := node.Execute(t.Context(), evalCtx, walkApply) + if !diags.HasErrors() { + t.Fatal("expected execute error, but there was none") + } + if got, want := diags.Err().Error(), "Output does not allow ephemeral value: The value that was generated for the output is ephemeral, but it is not configured to allow one"; !strings.Contains(got, want) { + t.Errorf("expected error to include %q, but was: %s", want, got) + } + }) + + t.Run("output marked as ephemeral in child module and receives a non-ephemeral value", func(t *testing.T) { + evalCtx := new(MockEvalContext) + evalCtx.StateState = states.NewState().SyncWrapper() + evalCtx.ChecksState = checks.NewState(nil) + + config := &configs.Output{Name: "map-output", Ephemeral: true} + addr := addrs.OutputValue{Name: config.Name}.Absolute(addrs.MustParseModuleInstanceStr("module.foo")) + node := &NodeApplyableOutput{Config: config, Addr: addr} + val := cty.MapVal(map[string]cty.Value{ + "a": cty.StringVal("b"), + }) + evalCtx.EvaluateExprResult = val + + diags := node.Execute(t.Context(), evalCtx, walkApply) + if diags.HasErrors() { + t.Fatalf("expected to have no errors") + } + gotVal := evalCtx.State().OutputValue(addr) + if !val.RawEquals(gotVal.Value) { + t.Fatalf("expected value is not in the state. expected: %+v; in state: %+v", val, gotVal.Value) + } + }) +} + func TestNodeApplyableOutputExecute_deprecatedOutput(t *testing.T) { evalCtx := new(MockEvalContext) evalCtx.StateState = states.NewState().SyncWrapper() diff --git a/internal/tofu/node_resource_abstract.go b/internal/tofu/node_resource_abstract.go index 9d55d499fc..f0f96c475a 100644 --- a/internal/tofu/node_resource_abstract.go +++ b/internal/tofu/node_resource_abstract.go @@ -89,7 +89,7 @@ type NodeAbstractResource struct { // Set from GraphNodeTargetable Excludes []addrs.Targetable - // Set from AttachDataResourceDependsOn + // Set from AttachResourceDependsOn dependsOn []addrs.ConfigResource forceDependsOn bool @@ -115,18 +115,18 @@ type NodeAbstractResource struct { } var ( - _ GraphNodeReferenceable = (*NodeAbstractResource)(nil) - _ GraphNodeReferencer = (*NodeAbstractResource)(nil) - _ GraphNodeProviderConsumer = (*NodeAbstractResource)(nil) - _ GraphNodeProvisionerConsumer = (*NodeAbstractResource)(nil) - _ GraphNodeConfigResource = (*NodeAbstractResource)(nil) - _ GraphNodeAttachResourceConfig = (*NodeAbstractResource)(nil) - _ GraphNodeAttachResourceSchema = (*NodeAbstractResource)(nil) - _ GraphNodeAttachProvisionerSchema = (*NodeAbstractResource)(nil) - _ GraphNodeAttachProviderMetaConfigs = (*NodeAbstractResource)(nil) - _ GraphNodeTargetable = (*NodeAbstractResource)(nil) - _ graphNodeAttachDataResourceDependsOn = (*NodeAbstractResource)(nil) - _ dag.GraphNodeDotter = (*NodeAbstractResource)(nil) + _ GraphNodeReferenceable = (*NodeAbstractResource)(nil) + _ GraphNodeReferencer = (*NodeAbstractResource)(nil) + _ GraphNodeProviderConsumer = (*NodeAbstractResource)(nil) + _ GraphNodeProvisionerConsumer = (*NodeAbstractResource)(nil) + _ GraphNodeConfigResource = (*NodeAbstractResource)(nil) + _ GraphNodeAttachResourceConfig = (*NodeAbstractResource)(nil) + _ GraphNodeAttachResourceSchema = (*NodeAbstractResource)(nil) + _ GraphNodeAttachProvisionerSchema = (*NodeAbstractResource)(nil) + _ GraphNodeAttachProviderMetaConfigs = (*NodeAbstractResource)(nil) + _ GraphNodeTargetable = (*NodeAbstractResource)(nil) + _ graphNodeAttachResourceDependsOn = (*NodeAbstractResource)(nil) + _ dag.GraphNodeDotter = (*NodeAbstractResource)(nil) ) // NewNodeAbstractResource creates an abstract resource graph node for @@ -455,8 +455,8 @@ func (n *NodeAbstractResource) SetExcludes(excludes []addrs.Targetable) { n.Excludes = excludes } -// graphNodeAttachDataResourceDependsOn -func (n *NodeAbstractResource) AttachDataResourceDependsOn(deps []addrs.ConfigResource, force bool) { +// graphNodeAttachResourceDependsOn +func (n *NodeAbstractResource) AttachResourceDependsOn(deps []addrs.ConfigResource, force bool) { n.dependsOn = deps n.forceDependsOn = force } diff --git a/internal/tofu/node_resource_abstract_instance.go b/internal/tofu/node_resource_abstract_instance.go index 3d20fbaa15..baece69134 100644 --- a/internal/tofu/node_resource_abstract_instance.go +++ b/internal/tofu/node_resource_abstract_instance.go @@ -10,6 +10,8 @@ import ( "fmt" "log" "strings" + "sync/atomic" + "time" "github.com/hashicorp/hcl/v2" "github.com/zclconf/go-cty/cty" @@ -81,6 +83,29 @@ type NodeAbstractResourceInstance struct { generatedConfigHCL string ResolvedProviderKey addrs.InstanceKey + + // These are the fields that should be strictly used when this node is acting upon an ephemeral resource. + // The ephemeralDiags and closeCh are initialized right before scheduling the renewal process. + // + // closeCh is the channel that will be close to stop the renewal goroutine. + // This is closed when the NodeAbstractResourceInstance.Close is called. NodeAbstractResourceInstance.Close will + // return immediately if renewStarted.Load() == false, meaning that the goroutine for ephemeral resource + // renewal never started. + // + // ephemeralDiags is used by the renewal goroutine to return whatever issues it encountered during the process. + // This is the channel that NodeAbstractResourceInstance.Close is blocking on, so be sure that when the goroutine + // is getting closed, there is something written into ephemeralDiags. Otherwise, NodeAbstractResourceInstance.Close + // will wait for a specific timeout before returning only a timeout diagnostic. + // The same channel is also used by the NodeAbstractResourceInstance.closeEphemeralResource to add the diagnostics + // that it encountered, if any. + // + // renewStarted is just used as a semaphore to be able to detect when an ephemeral resource renewal process didn't + // start so calls to NodeAbstractResourceInstance.Close can return no diagnostics whatsoever. + // A common reason for which the renewal goroutine can be skipped from being created is when the ephemeral + // resource is deferred for the apply phase. + closeCh chan struct{} + ephemeralDiags chan tfdiags.Diagnostics + renewStarted atomic.Bool } // NewNodeAbstractResourceInstance creates an abstract resource instance graph @@ -542,6 +567,7 @@ func (n *NodeAbstractResourceInstance) writeResourceInstanceStateImpl(ctx contex return fmt.Errorf("failed to encode %s in state: no resource type schema available", absAddr) } + obj.Value = schema.RemoveEphemeralFromWriteOnly(obj.Value) src, err := obj.Encode(schema.ImpliedType(), currentVersion) if err != nil { return fmt.Errorf("failed to encode %s in state: %w", absAddr, err) @@ -724,6 +750,8 @@ func (n *NodeAbstractResourceInstance) writeChange(ctx context.Context, evalCtx return fmt.Errorf("provider does not support resource type %q", ri.Resource.Type) } + change.Before = schema.RemoveEphemeralFromWriteOnly(change.Before) + change.After = schema.RemoveEphemeralFromWriteOnly(change.After) csrc, err := change.Encode(schema.ImpliedType()) if err != nil { return fmt.Errorf("failed to encode planned changes for %s: %w", n.Addr, err) @@ -952,7 +980,9 @@ func (n *NodeAbstractResourceInstance) plan( } origConfigVal, _, configDiags := evalCtx.EvaluateBlock(ctx, config.Config, schema, nil, keyData) - diags = diags.Append(configDiags) + // configDiags.InConfigBody(...) has been added after the initial implementation, to add + // additional context to the diagnostics generated by the ephemeral values references validation. + diags = diags.Append(configDiags.InConfigBody(config.Config, n.Addr.String())) if configDiags.HasErrors() { return nil, nil, keyData, diags } @@ -1797,6 +1827,130 @@ func (n *NodeAbstractResourceInstance) providerMetas(ctx context.Context, evalCt return metaConfigVal, diags } +func (n *NodeAbstractResourceInstance) openEphemeralResource(ctx context.Context, evalCtx EvalContext, configVal cty.Value) (cty.Value, providers.DeferralReason, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + var newVal cty.Value + + config := *n.Config + + provider, providerSchema, err := n.getProvider(ctx, evalCtx) + diags = diags.Append(err) + if diags.HasErrors() { + return newVal, providers.DeferredReasonUnknown, diags + } + schema, _ := providerSchema.SchemaForResourceAddr(n.Addr.ContainingResource().Resource) + if schema == nil { + // Should be caught during validation, so we don't bother with a pretty error here + diags = diags.Append(fmt.Errorf("provider %q does not support ephemeral resource %q", n.ResolvedProvider.ProviderConfig, n.Addr.ContainingResource().Resource.Type)) + return newVal, providers.DeferredReasonUnknown, diags + } + + // Unmark before sending to provider, will re-mark before returning + var pvm []cty.PathValueMarks + configVal, pvm = configVal.UnmarkDeepWithPaths() + + log.Printf("[TRACE] openEphemeralResource: Re-validating config for %s", n.Addr) + validateResp := provider.ValidateEphemeralConfig( + ctx, + providers.ValidateEphemeralConfigRequest{ + TypeName: n.Addr.ContainingResource().Resource.Type, + Config: configVal, + }, + ) + diags = diags.Append(validateResp.Diagnostics.InConfigBody(config.Config, n.Addr.String())) + if diags.HasErrors() { + return newVal, providers.DeferredReasonUnknown, diags + } + + // If we get down here then our configuration is complete and we're ready + // to actually call the provider to open the ephemeral resource. + log.Printf("[TRACE] openEphemeralResource: %s configuration is complete, so calling the provider", n.Addr) + + diags = diags.Append(evalCtx.Hook(func(h Hook) (HookAction, error) { + return h.PreOpen(n.Addr) + })) + if diags.HasErrors() { + return newVal, providers.DeferredReasonUnknown, diags + } + + req := providers.OpenEphemeralResourceRequest{ + TypeName: n.Addr.ContainingResource().Resource.Type, + Config: configVal, + } + resp := provider.OpenEphemeralResource(ctx, req) + diags = diags.Append(resp.Diagnostics.InConfigBody(config.Config, n.Addr.String())) + if diags.HasErrors() { + return newVal, providers.DeferredReasonUnknown, diags + } + + if resp.Deferred != nil { + return newVal, resp.Deferred.DeferralReason, diags + } + newVal = resp.Result + + for _, err := range newVal.Type().TestConformance(schema.ImpliedType()) { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Provider produced invalid object", + fmt.Sprintf( + "Provider %q produced an invalid value for %s.\n\nThis is a bug in the provider, which should be reported in the provider's own issue tracker.", + n.ResolvedProvider.ProviderConfig.InstanceString(n.ResolvedProviderKey), tfdiags.FormatErrorPrefixed(err, n.Addr.String()), + ), + )) + } + if diags.HasErrors() { + return newVal, providers.DeferredReasonUnknown, diags + } + + if newVal.IsNull() { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Provider produced null object", + fmt.Sprintf( + "Provider %q produced a null value for %s.\n\nThis is a bug in the provider, which should be reported in the provider's own issue tracker.", + n.ResolvedProvider.ProviderConfig.InstanceString(n.ResolvedProviderKey), n.Addr, + ), + )) + return newVal, providers.DeferredReasonUnknown, diags + } + + if !newVal.IsNull() && !newVal.IsWhollyKnown() { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Provider produced invalid object", + fmt.Sprintf( + "Provider %q produced a value for %s that is not wholly known.\n\nThis is a bug in the provider, which should be reported in the provider's own issue tracker.", + n.ResolvedProvider.ProviderConfig.InstanceString(n.ResolvedProviderKey), n.Addr, + ), + )) + return newVal, providers.DeferredReasonUnknown, diags + } + + if len(pvm) > 0 { + newVal = newVal.MarkWithPaths(pvm) + } + diags = diags.Append(evalCtx.Hook(func(h Hook) (HookAction, error) { + return h.PostOpen(n.Addr, diags.Err()) + })) + + // Initialize the closing channel and the channel that sends diagnostics back to the + // NodeAbstractResourceInstance.Close caller. + n.closeCh = make(chan struct{}, 1) + n.ephemeralDiags = make(chan tfdiags.Diagnostics, 1) + // Due to the go scheduler inner works, the goroutine spawned below can be actually scheduled + // later than the execution of the nodeCloseableResource graph node. + // Therefore, we want to mark the renewal process as started before the goroutine spawning to be sure + // that the execution of nodeCloseableResource will block on the diagnostics reported by the + // goroutine below. + n.renewStarted.Store(true) + // The renewer is taking care of calling provider.Renew if resp.RenewAt != nil. + // But if resp.RenewAt == nil, renewer holds only the resp.Private that will be used later + // when calling provider.CloseEphemeralResource. + go n.startEphemeralRenew(ctx, evalCtx, provider, resp.RenewAt, resp.Private) + + return newVal, providers.DeferredReasonUnknown, diags +} + // planDataSource deals with the main part of the data resource lifecycle: // either actually reading from the data source or generating a plan to do so. // @@ -1844,7 +1998,9 @@ func (n *NodeAbstractResourceInstance) planDataSource(ctx context.Context, evalC var configDiags tfdiags.Diagnostics configVal, _, configDiags = evalCtx.EvaluateBlock(ctx, config.Config, schema, nil, keyData) - diags = diags.Append(configDiags) + // configDiags.InConfigBody(...) has been added after the initial implementation, to add + // additional context to the diagnostics generated by the ephemeral values references validation. + diags = diags.Append(configDiags.InConfigBody(n.Config.Config, n.Addr.String())) if configDiags.HasErrors() { return nil, nil, keyData, diags } @@ -2029,6 +2185,8 @@ func (n *NodeAbstractResourceInstance) nestedInCheckBlock() (*configs.Check, boo // receiver depends on has a change pending in the plan, in which case we'd // need to override the usual behavior of immediately reading from the data // source where possible, and instead defer the read until the apply step. +// The deferral applies to the opening of the ephemeral resources, too, when +// the dependent managed resource is having pending changes. func (n *NodeAbstractResourceInstance) dependenciesHavePendingChanges(evalCtx EvalContext) bool { nModInst := n.Addr.Module nMod := nModInst.Module() @@ -2040,10 +2198,10 @@ func (n *NodeAbstractResourceInstance) dependenciesHavePendingChanges(evalCtx Ev depsToUse := n.dependsOn - if n.Addr.Resource.Resource.Mode == addrs.DataResourceMode { + if n.Addr.Resource.Resource.Mode == addrs.DataResourceMode || n.Addr.Resource.Resource.Mode == addrs.EphemeralResourceMode { if n.Config.HasCustomConditions() { - // For a data resource with custom conditions we need to look at - // the full set of resource dependencies -- both direct and + // For a data resource or an ephemeral resource with custom conditions + // we need to look at the full set of resource dependencies -- both direct and // indirect -- because an upstream update might be what's needed // in order to make a condition pass. depsToUse = n.Dependencies @@ -2051,13 +2209,23 @@ func (n *NodeAbstractResourceInstance) dependenciesHavePendingChanges(evalCtx Ev } for _, d := range depsToUse { - if d.Resource.Mode == addrs.DataResourceMode { + if n.Addr.Resource.Resource.Mode == addrs.DataResourceMode && d.Resource.Mode == addrs.DataResourceMode { // Data sources have no external side effects, so they pose a need // to delay this read. If they do have a change planned, it must be // because of a dependency on a managed resource, in which case // we'll also encounter it in this list of dependencies. continue } + if n.Addr.Resource.Resource.Mode == addrs.EphemeralResourceMode && d.Resource.Mode == addrs.EphemeralResourceMode { + // Ephemeral resources have no external side effects, so they pose a need + // to delay this opening. If they do have a change planned, it must be + // because of a dependency on a managed resource, in which case + // we'll also encounter it in this list of dependencies. + continue + } + // NOTE: a data source **can** have a depends_on entry that points to an ephemeral resource, but it cannot + // reference directly any attributes of an ephemeral resource. Therefore, we can encounter here an ephemeral resource + // being verified for changes. for _, change := range changes.GetChangesForConfigResource(d) { changeModInst := change.Addr.Module @@ -2382,6 +2550,17 @@ func (n *NodeAbstractResourceInstance) applyProvisioners(ctx context.Context, ev }) } } + // In case the configuration of a provisioner is referencing an + // ephemeral value, supress the whole output of the provisioner. + if _, hasEphemeral := configMarks[marks.Ephemeral]; hasEphemeral { + outputFn = func(msg string) { + // Given that we return nil below, this will never error + _ = evalCtx.Hook(func(h Hook) (HookAction, error) { + h.ProvisionOutput(n.Addr, prov.Type, "(output suppressed due to ephemeral value in config)") + return HookActionContinue, nil + }) + } + } output := CallbackUIOutput{OutputFn: outputFn} resp := provisioner.ProvisionResource(provisioners.ProvisionResourceRequest{ @@ -2882,3 +3061,329 @@ func maybeImproveResourceInstanceDiagnostics(diags tfdiags.Diagnostics, excludeA } return ret } + +func (n *NodeAbstractResourceInstance) applyEphemeralResource(ctx context.Context, evalCtx EvalContext) (*states.ResourceInstanceObject, instances.RepetitionData, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + var keyData instances.RepetitionData + var configVal cty.Value + + _, providerSchema, err := n.getProvider(ctx, evalCtx) + if err != nil { + return nil, keyData, diags.Append(err) + } + + schema, _ := providerSchema.SchemaForResourceAddr(n.Addr.ContainingResource().Resource) + if schema == nil { + // Should be caught during validation, so we don't bother with a pretty error here + diags = diags.Append(fmt.Errorf("provider %q does not support ephemeral resource %q", n.ResolvedProvider.ProviderConfig.InstanceString(n.ResolvedProviderKey), n.Addr.ContainingResource().Resource.Type)) + return nil, keyData, diags + } + + keyData = evalCtx.InstanceExpander().GetResourceInstanceRepetitionData(n.ResourceInstanceAddr()) + + var configDiags tfdiags.Diagnostics + configVal, _, configDiags = evalCtx.EvaluateBlock(ctx, n.Config.Config, schema, nil, keyData) + diags = diags.Append(configDiags) + if configDiags.HasErrors() { + return nil, keyData, diags + } + + configKnown := configVal.IsWhollyKnown() + // If our configuration contains any unknown values, or we depend on any + // unknown values then we must defer the opening to the apply phase by + // producing an "Open" change for this resource, and a placeholder value for + // it in the state. + if !configKnown { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Incomplete configuration for ephemeral resource", + Detail: fmt.Sprintf("Ephemeral resource %q has incomplete configuration.", n.Addr.String()), + Subject: n.Config.TypeRange.Ptr(), + Context: n.Config.DeclRange.Ptr(), + }) + return nil, instances.RepetitionData{}, diags + } + + // We have a complete configuration with no dependencies to wait on, so we + // can open the ephemeral resource and store its value in the state. + newVal, deferralReason, readDiags := n.openEphemeralResource(ctx, evalCtx, configVal) + if deferralReason != providers.DeferredReasonUnknown { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Ephemeral resource deferred during apply", + Detail: fmt.Sprintf("Ephemeral resource %q asked for being deferred. This is a provider error.", n.Addr.String()), + Subject: n.Config.TypeRange.Ptr(), + Context: n.Config.DeclRange.Ptr(), + }) + return nil, instances.RepetitionData{}, diags + } + diags = diags.Append(readDiags) + if diags.HasErrors() { + return nil, keyData, diags + } + // Now that we've loaded the data, and diags contain no error, + // we are going to create our proposedNewState. + plannedNewState := &states.ResourceInstanceObject{ + Value: newVal, + Status: states.ObjectReady, + // Private field ignored intentionally since this is handled internally by + // the goroutine that is handling the renewal of the ephemeral resource. + } + + return plannedNewState, keyData, diags +} + +func (n *NodeAbstractResourceInstance) planEphemeralResource(ctx context.Context, evalCtx EvalContext, checkRuleSeverity tfdiags.Severity, skipPlanChanges bool) (*plans.ResourceInstanceChange, *states.ResourceInstanceObject, instances.RepetitionData, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + var keyData instances.RepetitionData + var configVal cty.Value + + _, providerSchema, err := n.getProvider(ctx, evalCtx) + if err != nil { + return nil, nil, keyData, diags.Append(err) + } + + config := *n.Config + schema, _ := providerSchema.SchemaForResourceAddr(n.Addr.ContainingResource().Resource) + if schema == nil { + // Should be caught during validation, so we don't bother with a pretty error here + diags = diags.Append(fmt.Errorf("provider %q does not support ephemeral resource %q", n.ResolvedProvider.ProviderConfig.InstanceString(n.ResolvedProviderKey), n.Addr.ContainingResource().Resource.Type)) + return nil, nil, keyData, diags + } + + objTy := schema.ImpliedType() + priorVal := cty.NullVal(objTy) + + forEach, _ := evaluateForEachExpression(ctx, config.ForEach, evalCtx, n.Addr) + keyData = EvalDataForInstanceKey(n.ResourceInstanceAddr().Resource.Key, forEach) + + checkDiags := evalCheckRules( + ctx, + addrs.ResourcePrecondition, + n.Config.Preconditions, + evalCtx, n.Addr, keyData, + checkRuleSeverity, + ) + diags = diags.Append(checkDiags) + if diags.HasErrors() { + return nil, nil, keyData, diags // failed preconditions prevent further evaluation + } + + var configDiags tfdiags.Diagnostics + configVal, _, configDiags = evalCtx.EvaluateBlock(ctx, config.Config, schema, nil, keyData) + diags = diags.Append(configDiags) + if configDiags.HasErrors() { + return nil, nil, keyData, diags + } + + configKnown := configVal.IsWhollyKnown() + depsPending := n.dependenciesHavePendingChanges(evalCtx) + // If our configuration contains any unknown values, or we depend on any + // unknown values then we must defer the opening to the apply phase by + // producing an "Open" change for this resource, and a placeholder value for + // it in the state. + if depsPending || !configKnown { + // We can't plan any changes if we're only refreshing, so the only + // value we can set here is whatever was in state previously. + if skipPlanChanges { + plannedNewState := &states.ResourceInstanceObject{ + Value: priorVal, + Status: states.ObjectReady, + } + + return nil, plannedNewState, keyData, diags + } + + reason := "unknown reason" + if !configKnown { + log.Printf("[TRACE] planEphemeralResource: %s configuration not fully known yet, so deferring to apply phase", n.Addr) + reason = "unknown configuration" + } else if depsPending { + log.Printf("[TRACE] planEphemeralResource: %s configuration is fully known, at least one dependency has changes pending", n.Addr) + reason = "pending dependencies" + } + + plannedChange, plannedNewState, deferDiags := n.deferEphemeralResource(evalCtx, schema, priorVal, configVal, reason) + diags = diags.Append(deferDiags) + return plannedChange, plannedNewState, keyData, diags + } + + // We have a complete configuration with no dependencies to wait on, so we + // can open the ephemeral resource and store its value in the state. + newVal, deferralReason, readDiags := n.openEphemeralResource(ctx, evalCtx, configVal) + if deferralReason != providers.DeferredReasonUnknown { + reason := providers.DeferralReasonSummary(deferralReason) + + plannedChange, plannedNewState, deferDiags := n.deferEphemeralResource(evalCtx, schema, priorVal, configVal, reason) + diags = diags.Append(deferDiags) + return plannedChange, plannedNewState, keyData, diags + } + diags = diags.Append(readDiags) + if diags.HasErrors() { + return nil, nil, instances.RepetitionData{}, diags + } + + // Now we've loaded the data, and diags tells us whether we were successful + // or not, we are going to create our plannedChange and our + // proposedNewState. + plannedNewState := &states.ResourceInstanceObject{ + Value: newVal, + Status: states.ObjectReady, + // Private field ignored intentionally since this is handled internally by + // the goroutine that is handling the renewal of the ephemeral resource. + } + plannedChange := &plans.ResourceInstanceChange{ + Addr: n.Addr, + PrevRunAddr: n.Addr, + DeposedKey: states.NotDeposed, + ProviderAddr: n.ResolvedProvider.ProviderConfig, + Change: plans.Change{ + Action: plans.Open, + // In order to have proper evaluation of the references to ephemeral resources, we need the change to contain + // a proper after value that will be used later in evaluationStateData.GetResource to generate + // evaluation data of this resource. + // These values must not end up in the plan file. + // The nullification of these is handled at the plan file writing layer. + Before: priorVal, + After: newVal, + }, + } + + return plannedChange, plannedNewState, keyData, diags +} + +func (n *NodeAbstractResourceInstance) startEphemeralRenew(ctx context.Context, evalContext EvalContext, provider providers.Interface, renewAt *time.Time, privateData []byte) { + if n.Addr.Resource.Resource.Mode != addrs.EphemeralResourceMode { + panic("renewal process cannot be started for resources other than ephemeral ones. This is an OpenTofu issue, please report it") + } + privateData, diags := n.renewEphemeral(ctx, evalContext, provider, renewAt, privateData) + // wait for the close signal. This is like this because the renewEphemeral can return right away if the renewAt is nil. + // But the close of the ephemeral should happen only when the graph walk is reaching the execution of the closing + // ephemeral resource node. + <-n.closeCh + diags = diags.Append(n.closeEphemeralResource(ctx, evalContext, provider, privateData)) + n.ephemeralDiags <- diags +} + +func (n *NodeAbstractResourceInstance) closeEphemeralResource(ctx context.Context, evalContext EvalContext, provider providers.Interface, privateData []byte) (diags tfdiags.Diagnostics) { + req := providers.CloseEphemeralResourceRequest{ + TypeName: n.Addr.Resource.Resource.Type, + Private: privateData, + } + + // We are using cty.EmptyObject for the PreApply and PostApply because the prior state + // and the new planned state does not matter in ephemeral resources context, especially + // in the context of the close operation. + diags = diags.Append(evalContext.Hook(func(h Hook) (HookAction, error) { + return h.PreClose(n.Addr) + })) + resp := provider.CloseEphemeralResource(ctx, req) + diags = diags.Append(resp.Diagnostics) + + diags = diags.Append(evalContext.Hook(func(h Hook) (HookAction, error) { + return h.PostClose(n.Addr, diags.Err()) + })) + return diags.Append(diags) +} + +// renewEphemeral is meant to be called into a goroutine. This method listens on ctx.Done and n.closeCh for ending the job and +// to return the data. +func (n *NodeAbstractResourceInstance) renewEphemeral(ctx context.Context, evalContext EvalContext, provider providers.Interface, renewAt *time.Time, privateData []byte) ([]byte, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + for { + if renewAt == nil { + return privateData, diags + } + select { + case <-time.After(time.Until(*renewAt)): + case <-n.closeCh: + return privateData, diags + case <-ctx.Done(): + return privateData, diags + } + diags = diags.Append(evalContext.Hook(func(h Hook) (HookAction, error) { + // We are using cty.EmptyObject here because the prior state and the new planned state does not matter + // in ephemeral resources context, especially in the context of the renew operation. + return h.PreRenew(n.Addr) + })) + req := providers.RenewEphemeralResourceRequest{ + TypeName: n.Addr.Resource.Resource.Type, + Private: privateData, + } + resp := provider.RenewEphemeralResource(ctx, req) + diags = diags.Append(evalContext.Hook(func(h Hook) (HookAction, error) { + return h.PostRenew(n.Addr, diags.Err()) + })) + diags = diags.Append(resp.Diagnostics) + renewAt = resp.RenewAt + privateData = resp.Private + } +} + +// deferEphemeralResource is a helper function that builds a change and a state object by using a +// partial value and is announcing the deferral of the ephemeral resource. +func (n *NodeAbstractResourceInstance) deferEphemeralResource(evalCtx EvalContext, schema *configschema.Block, priorVal cty.Value, configVal cty.Value, reason string) ( + plannedChange *plans.ResourceInstanceChange, + plannedNewState *states.ResourceInstanceObject, + diags tfdiags.Diagnostics, +) { + + unmarkedConfigVal, configMarkPaths := configVal.UnmarkDeepWithPaths() + proposedNewVal := objchange.PlannedEphemeralResourceObject(schema, unmarkedConfigVal) + proposedNewVal = proposedNewVal.MarkWithPaths(configMarkPaths) + + plannedChange = &plans.ResourceInstanceChange{ + Addr: n.Addr, + PrevRunAddr: n.prevRunAddr(evalCtx), + ProviderAddr: n.ResolvedProvider.ProviderConfig, + Change: plans.Change{ + Action: plans.Open, + // In order to have proper evaluation of the references to ephemeral resources, we need the change to contain + // a proper after value, even if it's just a null value of the schema type. + // These values must not end up in the plan file. + // The nullification of these is handled at the plan file writing layer. + Before: priorVal, + After: proposedNewVal, + }, + // Skipped ActionReason on purpose since ephemeral resources changes are not meant + // to be shown in the UI. + } + + plannedNewState = &states.ResourceInstanceObject{ + Value: proposedNewVal, + Status: states.ObjectPlanned, + } + + diags = diags.Append(evalCtx.Hook(func(h Hook) (HookAction, error) { + return h.Deferred(n.Addr, reason) + })) + + return +} + +// Close is meant to be called against nodes representing ephemeral resources. +// When this is called, it will wait for the diagnostic responses that could have been +// returned during the Renew calls and return those back to the caller. +func (n *NodeAbstractResourceInstance) Close() tfdiags.Diagnostics { + if !n.renewStarted.Load() { + // If the ephemeral resource has been deferred, this method needs to return immediately. + return nil + } + defer func() { + close(n.ephemeralDiags) + n.renewStarted.Store(false) + }() + close(n.closeCh) + timeout := 10 * time.Second + select { + case d := <-n.ephemeralDiags: + return d + case <-time.After(timeout): + return tfdiags.Diagnostics{}.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Closing ephemeral resource timed out", + Detail: fmt.Sprintf("The ephemeral resource %q timed out on closing after %s", n.Addr.String(), timeout), + Subject: n.Config.DeclRange.Ptr(), + }) + } +} diff --git a/internal/tofu/node_resource_apply_instance.go b/internal/tofu/node_resource_apply_instance.go index 634a6333dd..1045fcd71c 100644 --- a/internal/tofu/node_resource_apply_instance.go +++ b/internal/tofu/node_resource_apply_instance.go @@ -174,6 +174,10 @@ func (n *NodeApplyableResourceInstance) Execute(ctx context.Context, evalCtx Eva diags = diags.Append( n.dataResourceExecute(ctx, evalCtx), ) + case addrs.EphemeralResourceMode: + diags = diags.Append( + n.ephemeralResourceExecute(ctx, evalCtx), + ) default: panic(fmt.Errorf("unsupported resource mode %s", n.Config.Mode)) } @@ -181,6 +185,34 @@ func (n *NodeApplyableResourceInstance) Execute(ctx context.Context, evalCtx Eva return diags } +func (n *NodeApplyableResourceInstance) ephemeralResourceExecute(ctx context.Context, evalCtx EvalContext) (diags tfdiags.Diagnostics) { + // For ephemeral resources we don't need the change or the state of it, that being used only + // for dependencies resolution and respectively, for expression evaluation. + state, repeatData, applyDiags := n.applyEphemeralResource(ctx, evalCtx) + diags = diags.Append(applyDiags) + if diags.HasErrors() { + return diags + } + + diags = diags.Append(n.writeResourceInstanceState(ctx, evalCtx, state, workingState)) + + // Post-conditions might block further progress. We intentionally do this + // _after_ writing the state/diff because we want to check against + // the result of the operation, and to fail on future operations + // until the user makes the condition succeed. + checkDiags := evalCheckRules( + ctx, + addrs.ResourcePostcondition, + n.Config.Postconditions, + evalCtx, n.ResourceInstanceAddr(), + repeatData, + tfdiags.Error, + ) + diags = diags.Append(checkDiags) + + return diags +} + func (n *NodeApplyableResourceInstance) dataResourceExecute(ctx context.Context, evalCtx EvalContext) (diags tfdiags.Diagnostics) { _, providerSchema, err := getProvider(ctx, evalCtx, n.ResolvedProvider.ProviderConfig, n.ResolvedProviderKey) diags = diags.Append(err) @@ -521,3 +553,11 @@ func maybeTainted(addr addrs.AbsResourceInstance, state *states.ResourceInstance } return state } + +// Close implements closableResource +func (n *NodeApplyableResourceInstance) Close() (diags tfdiags.Diagnostics) { + if n.Addr.Resource.Resource.Mode != addrs.EphemeralResourceMode { + return diags + } + return n.NodeAbstractResourceInstance.Close() +} diff --git a/internal/tofu/node_resource_closeable.go b/internal/tofu/node_resource_closeable.go new file mode 100644 index 0000000000..6a89c11926 --- /dev/null +++ b/internal/tofu/node_resource_closeable.go @@ -0,0 +1,58 @@ +// Copyright (c) The OpenTofu Authors +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2023 HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package tofu + +import ( + "context" + "log" + "sync" + + "github.com/opentofu/opentofu/internal/addrs" + "github.com/opentofu/opentofu/internal/tfdiags" +) + +type GraphNodeCloseableResource interface { + closeableSigil() +} + +// nodeCloseableResource is meant to just call the resourceCloser callback right before closing the providers. +// This is done this way strictly because all the information that it's needed to successfully handle ephemeral +// resources closing is in the node type that also opens it. +type nodeCloseableResource struct { + cbs []resourceCloser + Addr addrs.ConfigResource +} + +var ( + _ GraphNodeCloseableResource = (*nodeCloseableResource)(nil) +) + +func (n *nodeCloseableResource) Name() string { + return n.Addr.String() + " (close)" +} + +func (n *nodeCloseableResource) Execute(_ context.Context, _ EvalContext, _ walkOperation) (diags tfdiags.Diagnostics) { + var wg sync.WaitGroup + diagsCh := make(chan tfdiags.Diagnostics, len(n.cbs)) + log.Printf("[TRACE] nodeCloseableResource - scheduling %d closing operations for of ephemeral resource %s", len(n.cbs), n.Addr.String()) + // NOTE: since go v1.22 there is no need to copy the loop variable. + for _, cb := range n.cbs { + wg.Add(1) + go func() { + defer wg.Done() + diagsCh <- cb() + }() + } + wg.Wait() + close(diagsCh) + for d := range diagsCh { + diags = diags.Append(d) + } + return diags +} + +func (n *nodeCloseableResource) closeableSigil() { +} diff --git a/internal/tofu/node_resource_deposed.go b/internal/tofu/node_resource_deposed.go index 7fd1177bf2..4c6b02ed89 100644 --- a/internal/tofu/node_resource_deposed.go +++ b/internal/tofu/node_resource_deposed.go @@ -385,6 +385,8 @@ func (n *NodeDestroyDeposedResourceInstanceObject) writeResourceInstanceState(ct // fail to set up their world properly. return fmt.Errorf("failed to encode %s in state: no resource type schema available", absAddr) } + + obj.Value = schema.RemoveEphemeralFromWriteOnly(obj.Value) src, err := obj.Encode(schema.ImpliedType(), currentVersion) if err != nil { return fmt.Errorf("failed to encode %s in state: %w", absAddr, err) diff --git a/internal/tofu/node_resource_destroy.go b/internal/tofu/node_resource_destroy.go index 2e554a0c24..addb1da082 100644 --- a/internal/tofu/node_resource_destroy.go +++ b/internal/tofu/node_resource_destroy.go @@ -55,10 +55,20 @@ func (n *NodeDestroyResourceInstance) Name() string { } func (n *NodeDestroyResourceInstance) ProvidedBy() RequestedProvider { - if n.Addr.Resource.Resource.Mode == addrs.DataResourceMode { + switch n.Addr.Resource.Resource.Mode { + case addrs.DataResourceMode: // indicate that this node does not require a configured provider return RequestedProvider{} + case addrs.EphemeralResourceMode: + // Since ephemeral resources are not stored into the state or plan files, + // a change of type delete cannot be generated for it, meaning that this + // code path is not meant to be reached. + // Even though, let's ensure that ever the case, a destroy node for an + // ephemeral resource indicates correctly that for its removal there + // is no provider needed. + return RequestedProvider{} } + return n.NodeAbstractResourceInstance.ProvidedBy() } @@ -171,6 +181,10 @@ func (n *NodeDestroyResourceInstance) Execute(ctx context.Context, evalCtx EvalC diags = diags.Append( n.dataResourceExecute(ctx, evalCtx), ) + case addrs.EphemeralResourceMode: + diags = diags.Append( + n.ephemeralResourceExecute(ctx, evalCtx), + ) default: panic(fmt.Errorf("unsupported resource mode %s", n.Config.Mode)) } @@ -264,3 +278,16 @@ func (n *NodeDestroyResourceInstance) dataResourceExecute(_ context.Context, eva evalCtx.State().SetResourceInstanceCurrent(n.Addr, nil, n.ResolvedProvider.ProviderConfig, n.ResolvedProviderKey) return diags.Append(updateStateHook(evalCtx)) } + +// ephemeralResourceExecute for NodeDestroyResourceInstance is only here to return an error. +// An ephemeral resource, by definition, cannot be destroyed. If the execution path is reaching this part, it means that +// there is an issue somewhere else, most probably in the planning phase since the generation of NodeDestroyResourceInstance +// is strictly related to the changes from the plan. +func (n *NodeDestroyResourceInstance) ephemeralResourceExecute(_ context.Context, _ EvalContext) (diags tfdiags.Diagnostics) { + log.Printf("[TRACE] NodeDestroyResourceInstance: called for ephemeral resource %s", n.Addr) + return diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Destroy invoked for an ephemeral resource", + fmt.Sprintf("A destroy operation has been invoked for the ephemeral resource %q. This is an OpenTofu error. Please report this.", n.Addr), + )) +} diff --git a/internal/tofu/node_resource_plan.go b/internal/tofu/node_resource_plan.go index 1126469ee4..e0c8881910 100644 --- a/internal/tofu/node_resource_plan.go +++ b/internal/tofu/node_resource_plan.go @@ -8,7 +8,9 @@ package tofu import ( "context" "fmt" + "log" "strings" + "sync" "github.com/opentofu/opentofu/internal/addrs" "github.com/opentofu/opentofu/internal/dag" @@ -48,6 +50,12 @@ type nodeExpandPlannableResource struct { // structure in the future, as we need to compare for equality and take the // union of multiple groups of dependencies. dependencies []addrs.ConfigResource + + // This slice is meant to keep references to the resourceCloser's of the expanded instances. + // Later, this will be called from nodeCloseableResource. + // At the time of introducing this, it was strictly meant for ephemeral resources, but if there + // will be other closeable resources, this could be used for those too. + closers []resourceCloser } var ( @@ -60,6 +68,7 @@ var ( _ GraphNodeAttachDependencies = (*nodeExpandPlannableResource)(nil) _ GraphNodeTargetable = (*nodeExpandPlannableResource)(nil) _ graphNodeExpandsInstances = (*nodeExpandPlannableResource)(nil) + _ closableResource = (*nodeExpandPlannableResource)(nil) ) func (n *nodeExpandPlannableResource) Name() string { @@ -378,6 +387,11 @@ func (n *nodeExpandPlannableResource) resourceInstanceSubgraph(ctx context.Conte if resolvedImportTarget != nil { m.importTarget = *resolvedImportTarget } + // When creating concrete instance nodes for the ephemeral resources we want to collect all the + // resourceCloser callbacks from the nodes to be able to close the resources at the end of the graph walk. + if a.Addr.Resource.Resource.Mode == addrs.EphemeralResourceMode { + n.closers = append(n.closers, m.Close) + } return m } @@ -438,3 +452,28 @@ func (n *nodeExpandPlannableResource) resourceInstanceSubgraph(ctx context.Conte graph, graphDiags := b.Build(ctx, addr.Module) return graph, diags.Append(graphDiags).ErrWithWarnings() } + +// Close implements closableResource +func (n *nodeExpandPlannableResource) Close() (diags tfdiags.Diagnostics) { + if n.Addr.Resource.Mode != addrs.EphemeralResourceMode { + return diags + } + + var wg sync.WaitGroup + diagsCh := make(chan tfdiags.Diagnostics, len(n.closers)) + log.Printf("[TRACE] nodeExpandPlannableResource - scheduling %d closing operations for of ephemeral resource %s", len(n.closers), n.Addr.String()) + // NOTE: since go v1.22 there is no need to copy the loop variable. + for _, cb := range n.closers { + wg.Add(1) + go func() { + defer wg.Done() + diagsCh <- cb() + }() + } + wg.Wait() + close(diagsCh) + for d := range diagsCh { + diags = diags.Append(d) + } + return diags +} diff --git a/internal/tofu/node_resource_plan_destroy.go b/internal/tofu/node_resource_plan_destroy.go index 195bb55878..a56bf4dd9a 100644 --- a/internal/tofu/node_resource_plan_destroy.go +++ b/internal/tofu/node_resource_plan_destroy.go @@ -8,7 +8,9 @@ package tofu import ( "context" "fmt" + "log" + "github.com/opentofu/opentofu/internal/dag" otelAttr "go.opentelemetry.io/otel/attribute" otelTrace "go.opentelemetry.io/otel/trace" @@ -40,8 +42,14 @@ var ( _ GraphNodeAttachResourceState = (*NodePlanDestroyableResourceInstance)(nil) _ GraphNodeExecutable = (*NodePlanDestroyableResourceInstance)(nil) _ GraphNodeProviderConsumer = (*NodePlanDestroyableResourceInstance)(nil) + _ dag.NamedVertex = (*NodePlanDestroyableResourceInstance)(nil) ) +// dag.NamedVertex +func (n *NodePlanDestroyableResourceInstance) Name() string { + return n.NodeAbstractResourceInstance.Name() + " (destroy)" +} + // GraphNodeDestroyer func (n *NodePlanDestroyableResourceInstance) DestroyAddr() *addrs.AbsResourceInstance { addr := n.ResourceInstanceAddr() @@ -79,6 +87,10 @@ func (n *NodePlanDestroyableResourceInstance) Execute(ctx context.Context, evalC diags = diags.Append( n.dataResourceExecute(ctx, evalCtx, op), ) + case addrs.EphemeralResourceMode: + diags = diags.Append( + n.ephemeralResourceExecute(ctx, evalCtx, op), + ) default: panic(fmt.Errorf("unsupported resource mode %s", n.Config.Mode)) } @@ -154,3 +166,12 @@ func (n *NodePlanDestroyableResourceInstance) dataResourceExecute(ctx context.Co } return diags.Append(n.writeChange(ctx, evalCtx, change, "")) } + +func (n *NodePlanDestroyableResourceInstance) ephemeralResourceExecute(_ context.Context, _ EvalContext, _ walkOperation) (diags tfdiags.Diagnostics) { + log.Printf("[TRACE] NodePlanDestroyableResourceInstance: called for ephemeral resource %s", n.Addr) + return diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "An ephemeral resource planned for destroy", + fmt.Sprintf("A destroy operation has been planned for the ephemeral resource %q. This is an OpenTofu error. Please report this.", n.Addr), + )) +} diff --git a/internal/tofu/node_resource_plan_instance.go b/internal/tofu/node_resource_plan_instance.go index cb0da3ad4c..ee49b923c7 100644 --- a/internal/tofu/node_resource_plan_instance.go +++ b/internal/tofu/node_resource_plan_instance.go @@ -119,6 +119,10 @@ func (n *NodePlannableResourceInstance) Execute(ctx context.Context, evalCtx Eva diags = diags.Append( n.dataResourceExecute(ctx, evalCtx), ) + case addrs.EphemeralResourceMode: + diags = diags.Append( + n.ephemeralResourceExecute(ctx, evalCtx), + ) default: panic(fmt.Errorf("unsupported resource mode %s", n.Config.Mode)) } @@ -183,6 +187,59 @@ func (n *NodePlannableResourceInstance) dataResourceExecute(ctx context.Context, return diags } +func (n *NodePlannableResourceInstance) ephemeralResourceExecute(ctx context.Context, evalCtx EvalContext) (diags tfdiags.Diagnostics) { + config := n.Config + addr := n.ResourceInstanceAddr() + + _, providerSchema, err := getProvider(ctx, evalCtx, n.ResolvedProvider.ProviderConfig, n.ResolvedProviderKey) + diags = diags.Append(err) + if diags.HasErrors() { + return diags + } + + diags = diags.Append(validateSelfRef(addr.Resource, config.Config, providerSchema)) + if diags.HasErrors() { + return diags + } + + checkRuleSeverity := tfdiags.Error + if n.skipPlanChanges || n.preDestroyRefresh { + checkRuleSeverity = tfdiags.Warning + } + + change, state, repeatData, planDiags := n.planEphemeralResource(ctx, evalCtx, checkRuleSeverity, n.skipPlanChanges) + diags = diags.Append(planDiags) + if diags.HasErrors() { + return diags + } + + // We are writing the changes of the ephemeral resource to make it visible + // to any other ephemeral resource that might depend on this one. + // This is needed to determine if an ephemeral resource should be deferred or not. + diags = diags.Append(n.writeChange(ctx, evalCtx, change, states.NotDeposed)) + // write ephemeral resource only in the working state to make it accessible to the evaluator. + // This is later filtered out when it comes to the state or plan writing. + diags = diags.Append(n.writeResourceInstanceState(ctx, evalCtx, state, workingState)) + if diags.HasErrors() { + return diags + } + + // Post-conditions might block further progress. We intentionally do this + // _after_ writing the state/diff because we want to check against + // the result of the operation, and to fail on future operations + // until the user makes the condition succeed. + checkDiags := evalCheckRules( + ctx, + addrs.ResourcePostcondition, + n.Config.Postconditions, + evalCtx, addr, repeatData, + checkRuleSeverity, + ) + diags = diags.Append(checkDiags) + + return diags +} + func (n *NodePlannableResourceInstance) managedResourceExecute(ctx context.Context, evalCtx EvalContext) (diags tfdiags.Diagnostics) { config := n.Config addr := n.ResourceInstanceAddr() diff --git a/internal/tofu/node_resource_plan_orphan.go b/internal/tofu/node_resource_plan_orphan.go index 1bdd21e90d..9130147a6f 100644 --- a/internal/tofu/node_resource_plan_orphan.go +++ b/internal/tofu/node_resource_plan_orphan.go @@ -91,6 +91,10 @@ func (n *NodePlannableResourceInstanceOrphan) Execute(ctx context.Context, evalC diags = diags.Append( n.dataResourceExecute(ctx, evalCtx), ) + case addrs.EphemeralResourceMode: + diags = diags.Append( + n.ephemeralResourceExecute(ctx, evalCtx), + ) default: panic(fmt.Errorf("unsupported resource mode %s", n.Config.Mode)) } @@ -99,8 +103,16 @@ func (n *NodePlannableResourceInstanceOrphan) Execute(ctx context.Context, evalC } func (n *NodePlannableResourceInstanceOrphan) ProvidedBy() RequestedProvider { - if n.Addr.Resource.Resource.Mode == addrs.DataResourceMode { - // indicate that this node does not require a configured provider + switch n.Addr.Resource.Resource.Mode { + case addrs.DataResourceMode: + return RequestedProvider{} + case addrs.EphemeralResourceMode: + // Since ephemeral resources are not stored into the state or plan files, such resources cannot be marked as orphan. + // In other words, this code path should never be reached since an orphan node for an ephemeral resource should not be + // possible to be created. + // Even though this is not possible, let's ensure that we are handling this accordingly, to not create unwanted behavior. + // If an ephemeral resource will ever have an orphan node that will be executed, an error will be raised + // from NodePlannableResourceInstanceOrphan#Execute(). return RequestedProvider{} } return n.NodeAbstractResourceInstance.ProvidedBy() @@ -353,3 +365,12 @@ func (n *NodePlannableResourceInstanceOrphan) deleteActionReason(evalCtx EvalCon // without any explicit reasoning. return plans.ResourceInstanceChangeNoReason } + +func (n *NodePlannableResourceInstanceOrphan) ephemeralResourceExecute(_ context.Context, _ EvalContext) (diags tfdiags.Diagnostics) { + log.Printf("[TRACE] NodePlannableResourceInstanceOrphan: called for ephemeral resource %s", n.Addr) + return diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "An ephemeral resource registered as orphan", + fmt.Sprintf("Ephemeral resource %q registered as orphan. This is an OpenTofu error. Please report this.", n.Addr), + )) +} diff --git a/internal/tofu/node_resource_plan_orphan_test.go b/internal/tofu/node_resource_plan_orphan_test.go index b5296d7138..1fafc4c5af 100644 --- a/internal/tofu/node_resource_plan_orphan_test.go +++ b/internal/tofu/node_resource_plan_orphan_test.go @@ -402,3 +402,27 @@ func TestNodeResourcePlanOrphanExecute_deposed(t *testing.T) { t.Fatalf("unexpected error: %s", diags.Err()) } } + +// TestNodeResourcePlanOrphanExecute_ephemeral is validating that NodePlannableResourceInstanceOrphan.Execute +// returns the expected error when it's executed against an ephemeral resource. +// Since ephemeral resources are not meant to be in the state, this case is not really possible in real life. +func TestNodeResourcePlanOrphanExecute_ephemeral(t *testing.T) { + addr := addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_object", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance) + + node := NodePlannableResourceInstanceOrphan{ + NodeAbstractResourceInstance: &NodeAbstractResourceInstance{ + NodeAbstractResource: NodeAbstractResource{}, + Addr: addr, + }, + } + diags := node.Execute(t.Context(), nil, walkPlan) + got := diags.Err().Error() + want := `An ephemeral resource registered as orphan: Ephemeral resource "ephemeral.test_object.foo" registered as orphan. This is an OpenTofu error. Please report this.` + if got != want { + t.Fatalf("unexpected error returned.\ngot: %s\nwant:%s", got, want) + } +} diff --git a/internal/tofu/node_resource_validate.go b/internal/tofu/node_resource_validate.go index fac3ca4983..63b25fe9fa 100644 --- a/internal/tofu/node_resource_validate.go +++ b/internal/tofu/node_resource_validate.go @@ -238,19 +238,7 @@ func (n *NodeValidatableResource) validateResource(ctx context.Context, evalCtx case addrs.ManagedResourceMode: schema, _ := providerSchema.SchemaForResourceType(n.Config.Mode, n.Config.Type) if schema == nil { - var suggestion string - if dSchema, _ := providerSchema.SchemaForResourceType(addrs.DataResourceMode, n.Config.Type); dSchema != nil { - suggestion = fmt.Sprintf("\n\nDid you intend to use the data source %q? If so, declare this using a \"data\" block instead of a \"resource\" block.", n.Config.Type) - } else if len(providerSchema.ResourceTypes) > 0 { - suggestions := make([]string, 0, len(providerSchema.ResourceTypes)) - for name := range providerSchema.ResourceTypes { - suggestions = append(suggestions, name) - } - if suggestion = didyoumean.NameSuggestion(n.Config.Type, suggestions); suggestion != "" { - suggestion = fmt.Sprintf(" Did you mean %q?", suggestion) - } - } - + suggestion := n.noResourceSchemaSuggestion(providerSchema) diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid resource type", @@ -261,7 +249,7 @@ func (n *NodeValidatableResource) validateResource(ctx context.Context, evalCtx } configVal, _, valDiags := evalCtx.EvaluateBlock(ctx, n.Config.Config, schema, nil, keyData) - diags = diags.Append(valDiags) + diags = diags.Append(valDiags.InConfigBody(n.Config.Config, n.Addr.String())) if valDiags.HasErrors() { return diags } @@ -312,19 +300,7 @@ func (n *NodeValidatableResource) validateResource(ctx context.Context, evalCtx case addrs.DataResourceMode: schema, _ := providerSchema.SchemaForResourceType(n.Config.Mode, n.Config.Type) if schema == nil { - var suggestion string - if dSchema, _ := providerSchema.SchemaForResourceType(addrs.ManagedResourceMode, n.Config.Type); dSchema != nil { - suggestion = fmt.Sprintf("\n\nDid you intend to use the managed resource type %q? If so, declare this using a \"resource\" block instead of a \"data\" block.", n.Config.Type) - } else if len(providerSchema.DataSources) > 0 { - suggestions := make([]string, 0, len(providerSchema.DataSources)) - for name := range providerSchema.DataSources { - suggestions = append(suggestions, name) - } - if suggestion = didyoumean.NameSuggestion(n.Config.Type, suggestions); suggestion != "" { - suggestion = fmt.Sprintf(" Did you mean %q?", suggestion) - } - } - + suggestion := n.noResourceSchemaSuggestion(providerSchema) diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid data source", @@ -334,6 +310,34 @@ func (n *NodeValidatableResource) validateResource(ctx context.Context, evalCtx return diags } + configVal, _, valDiags := evalCtx.EvaluateBlock(ctx, n.Config.Config, schema, nil, keyData) + diags = diags.Append(valDiags.InConfigBody(n.Config.Config, n.Addr.String())) + if valDiags.HasErrors() { + return diags + } + + // Use unmarked value for validate request + unmarkedConfigVal, _ := configVal.UnmarkDeep() + req := providers.ValidateDataResourceConfigRequest{ + TypeName: n.Config.Type, + Config: unmarkedConfigVal, + } + + resp := provider.ValidateDataResourceConfig(ctx, req) + diags = diags.Append(resp.Diagnostics.InConfigBody(n.Config.Config, n.Addr.String())) + case addrs.EphemeralResourceMode: + schema, _ := providerSchema.SchemaForResourceType(n.Config.Mode, n.Config.Type) + if schema == nil { + suggestion := n.noResourceSchemaSuggestion(providerSchema) + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid ephemeral resource", + Detail: fmt.Sprintf("The provider %s does not support ephemeral resource %q.%s", n.Provider().ForDisplay(), n.Config.Type, suggestion), + Subject: &n.Config.TypeRange, + }) + return diags + } + configVal, _, valDiags := evalCtx.EvaluateBlock(ctx, n.Config.Config, schema, nil, keyData) diags = diags.Append(valDiags) if valDiags.HasErrors() { @@ -342,12 +346,12 @@ func (n *NodeValidatableResource) validateResource(ctx context.Context, evalCtx // Use unmarked value for validate request unmarkedConfigVal, _ := configVal.UnmarkDeep() - req := providers.ValidateDataResourceConfigRequest{ + req := providers.ValidateEphemeralConfigRequest{ TypeName: n.Config.Type, Config: unmarkedConfigVal, } - resp := provider.ValidateDataResourceConfig(ctx, req) + resp := provider.ValidateEphemeralConfig(ctx, req) diags = diags.Append(resp.Diagnostics.InConfigBody(n.Config.Config, n.Addr.String())) } @@ -364,7 +368,53 @@ func (n *NodeValidatableResource) validateImportIDs(ctx context.Context, evalCtx } } return diags +} +// noResourceSchemaSuggestion is trying to generate a suggestion to be appended into the diagnostic that is pointing to the fact +// that the resource indicated by the user does not exist. This is doing its best to find a better alternative: +// - It is checking if in the provider's schema exists a resource with the same resource type but with a different mode. +// - If none found at the step above, it tries to determine if the name of the resource is incomplete and tries to recommend the +// closest resource type name to the one that is already configured. +func (n *NodeValidatableResource) noResourceSchemaSuggestion(providerSchema providers.ProviderSchema) string { + var suggestion string + if candidateMode, candidateSchema := nodeValidationAlternateBlockModeSuggestion(providerSchema, n.Config.Mode, n.Config.Type); candidateSchema != nil { + suggestion = fmt.Sprintf("\n\nDid you intend to use a block of type %q %q? If so, declare this using a block of type %q instead of one of type %q.", + addrs.ResourceModeBlockName(candidateMode), n.Config.Type, addrs.ResourceModeBlockName(candidateMode), addrs.ResourceModeBlockName(n.Config.Mode)) + } else if len(providerSchema.ResourceTypes) > 0 { + suggestions := make([]string, 0, len(providerSchema.ResourceTypes)) + for name := range providerSchema.ResourceTypes { + suggestions = append(suggestions, name) + } + if suggestion = didyoumean.NameSuggestion(n.Config.Type, suggestions); suggestion != "" { + suggestion = fmt.Sprintf(" Did you mean %q?", suggestion) + } + } + return suggestion +} + +// nodeValidationAlternateBlockModeSuggestion is trying to find an alternative addrs.ResourceMode for the given resourceType in the provider's schema. +// This is needed to be able to provide a suggestion when the user is using a wrong block type for the type of the resource that it's intended +// to be used. +func nodeValidationAlternateBlockModeSuggestion(schema providers.ProviderSchema, mode addrs.ResourceMode, resourceType string) (addrs.ResourceMode, *configschema.Block) { + filterOnOtherModes := func(targetModes []addrs.ResourceMode) (addrs.ResourceMode, *configschema.Block) { + for _, candidateMode := range targetModes { + if b, _ := schema.SchemaForResourceType(candidateMode, resourceType); b != nil { + return candidateMode, b + } + } + return addrs.InvalidResourceMode, nil + } + + switch mode { + case addrs.ManagedResourceMode: + return filterOnOtherModes([]addrs.ResourceMode{addrs.DataResourceMode, addrs.EphemeralResourceMode}) + case addrs.DataResourceMode: + return filterOnOtherModes([]addrs.ResourceMode{addrs.ManagedResourceMode, addrs.EphemeralResourceMode}) + case addrs.EphemeralResourceMode: + return filterOnOtherModes([]addrs.ResourceMode{addrs.ManagedResourceMode, addrs.DataResourceMode}) + } + + return addrs.InvalidResourceMode, nil } func (n *NodeValidatableResource) evaluateExpr(ctx context.Context, evalCtx EvalContext, expr hcl.Expression, wantTy cty.Type, self addrs.Referenceable, keyData instances.RepetitionData) (cty.Value, tfdiags.Diagnostics) { diff --git a/internal/tofu/node_resource_validate_test.go b/internal/tofu/node_resource_validate_test.go index ef11b6acab..384cd9c7b0 100644 --- a/internal/tofu/node_resource_validate_test.go +++ b/internal/tofu/node_resource_validate_test.go @@ -7,6 +7,7 @@ package tofu import ( "errors" + "fmt" "strings" "testing" @@ -273,6 +274,84 @@ func TestNodeValidatableResource_ValidateResource_managedResourceCount(t *testin } } +func TestNodeValidatableResource_ValidateResource_ephemeralResource(t *testing.T) { + mp := simpleMockProvider() + + p := providers.Interface(mp) + rc := &configs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "test_object", + Name: "foo", + Config: configs.SynthBody("", map[string]cty.Value{ + "test_string": cty.StringVal("bar"), + "test_number": cty.NumberIntVal(2).Mark(marks.Sensitive), + }), + } + + node := NodeValidatableResource{ + NodeAbstractResource: &NodeAbstractResource{ + Addr: mustConfigResourceAddr("ephemeral.test_foo.bar"), + Config: rc, + ResolvedProvider: ResolvedProvider{ProviderConfig: mustProviderConfig(`provider["registry.opentofu.org/hashicorp/aws"]`)}, + }, + } + + ctx := &MockEvalContext{} + ctx.installSimpleEval() + ctx.ProviderSchemaSchema = mp.GetProviderSchema(t.Context()) + ctx.ProviderProvider = p + + t.Run("no errors", func(t *testing.T) { + mp.ValidateEphemeralConfigCalled = false + mp.ValidateEphemeralConfigFn = func(req providers.ValidateEphemeralConfigRequest) providers.ValidateEphemeralConfigResponse { + if got, want := req.TypeName, "test_object"; got != want { + t.Fatalf("wrong resource type\ngot: %#v\nwant: %#v", got, want) + } + if got, want := req.Config.GetAttr("test_string"), cty.StringVal("bar"); !got.RawEquals(want) { + t.Fatalf("wrong value for test_string\ngot: %#v\nwant: %#v", got, want) + } + if got, want := req.Config.GetAttr("test_number"), cty.NumberIntVal(2); !got.RawEquals(want) { + t.Fatalf("wrong value for test_number\ngot: %#v\nwant: %#v", got, want) + } + return providers.ValidateEphemeralConfigResponse{} + } + diags := node.validateResource(t.Context(), ctx) + if diags.HasErrors() { + t.Fatalf("err: %s", diags.Err()) + } + + if !mp.ValidateEphemeralConfigCalled { + t.Fatal("Expected ValidateEphemeralResourceConfig to be called, but it was not!") + } + }) + t.Run("validation diagnostics returned", func(t *testing.T) { + mp.ValidateEphemeralConfigCalled = false + mp.ValidateEphemeralConfigFn = func(req providers.ValidateEphemeralConfigRequest) providers.ValidateEphemeralConfigResponse { + return providers.ValidateEphemeralConfigResponse{ + Diagnostics: tfdiags.Diagnostics{}.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "validation summary", + Detail: "validation detail", + }), + } + } + + diags := node.validateResource(t.Context(), ctx) + if !mp.ValidateEphemeralConfigCalled { + t.Fatal("Expected ValidateEphemeralResourceConfig to be called, but it was not!") + } + if !diags.HasErrors() { + t.Fatalf("expected err. Got nothing") + } + if got, want := diags[0].Description().Summary, "validation summary"; got != want { + t.Fatalf("expected diagnostic summary %q but got %q", want, got) + } + if got, want := diags[0].Description().Detail, "validation detail"; got != want { + t.Fatalf("expected diagnostic detail %q but got %q", want, got) + } + }) +} + func TestNodeValidatableResource_ValidateResource_dataSource(t *testing.T) { mp := simpleMockProvider() mp.ValidateDataResourceConfigFn = func(req providers.ValidateDataResourceConfigRequest) providers.ValidateDataResourceConfigResponse { @@ -638,3 +717,123 @@ The attribute computed_string is decided by the provider alone and therefore the t.Fatalf("wrong error\ngot: %s\nwant: Message containing %q", got, want) } } + +// TestNodeValidatableResource_ValidateResource_suggestion validates that the suggestions generated for mismatched block types +// are generated correctly. This was added when ephemeral resource were added and the suggestion code refactored to accommodate +// all resource modes in one. +func TestNodeValidatableResource_ValidateResource_suggestion(t *testing.T) { + ms := &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "test_string": { + Type: cty.String, + Optional: true, + }, + "computed_string": { + Type: cty.String, + Computed: true, + Optional: false, + }, + }, + } + + mp := &MockProvider{ + GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ + Provider: providers.Schema{Block: ms}, + ResourceTypes: map[string]providers.Schema{ + "test_managed_resource": {Block: ms}, + }, + DataSources: map[string]providers.Schema{ + "test_data_source": {Block: ms}, + }, + EphemeralResources: map[string]providers.Schema{ + "test_ephemeral_resource": {Block: ms}, + }, + }, + } + + tests := map[string]struct { + mode addrs.ResourceMode + rtype string + + wantSummary string + wantDetail string + }{ + "managed resource with resource type of a data source": { + mode: addrs.ManagedResourceMode, + rtype: "test_data_source", + wantSummary: "Invalid resource type", + wantDetail: "The provider hashicorp/aws does not support resource type \"test_data_source\".\n\nDid you intend to use a block of type \"data\" \"test_data_source\"? If so, declare this using a block of type \"data\" instead of one of type \"resource\".", + }, + "managed resource with resource type of an ephemeral resource": { + mode: addrs.ManagedResourceMode, + rtype: "test_ephemeral_resource", + wantSummary: "Invalid resource type", + wantDetail: "The provider hashicorp/aws does not support resource type \"test_ephemeral_resource\".\n\nDid you intend to use a block of type \"ephemeral\" \"test_ephemeral_resource\"? If so, declare this using a block of type \"ephemeral\" instead of one of type \"resource\".", + }, + "data source with resource type of a managed resource": { + mode: addrs.DataResourceMode, + rtype: "test_managed_resource", + wantSummary: "Invalid data source", + wantDetail: "The provider hashicorp/aws does not support data source \"test_managed_resource\".\n\nDid you intend to use a block of type \"resource\" \"test_managed_resource\"? If so, declare this using a block of type \"resource\" instead of one of type \"data\".", + }, + "data source with resource type of an ephemeral resource": { + mode: addrs.DataResourceMode, + rtype: "test_ephemeral_resource", + wantSummary: "Invalid data source", + wantDetail: "The provider hashicorp/aws does not support data source \"test_ephemeral_resource\".\n\nDid you intend to use a block of type \"ephemeral\" \"test_ephemeral_resource\"? If so, declare this using a block of type \"ephemeral\" instead of one of type \"data\".", + }, + "ephemeral resource with resource type of a managed resource": { + mode: addrs.EphemeralResourceMode, + rtype: "test_managed_resource", + wantSummary: "Invalid ephemeral resource", + wantDetail: "The provider hashicorp/aws does not support ephemeral resource \"test_managed_resource\".\n\nDid you intend to use a block of type \"resource\" \"test_managed_resource\"? If so, declare this using a block of type \"resource\" instead of one of type \"ephemeral\".", + }, + "ephemeral resource with resource type of a data source": { + mode: addrs.EphemeralResourceMode, + rtype: "test_data_source", + wantSummary: "Invalid ephemeral resource", + wantDetail: "The provider hashicorp/aws does not support ephemeral resource \"test_data_source\".\n\nDid you intend to use a block of type \"data\" \"test_data_source\"? If so, declare this using a block of type \"data\" instead of one of type \"ephemeral\".", + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + p := providers.Interface(mp) + rc := &configs.Resource{ + Mode: tt.mode, + Type: tt.rtype, + Name: "foo", + Config: configs.SynthBody("", map[string]cty.Value{}), + } + var prefix string + if tt.mode != addrs.ManagedResourceMode { + prefix = addrs.ResourceModeBlockName(tt.mode) + } + node := NodeValidatableResource{ + NodeAbstractResource: &NodeAbstractResource{ + Addr: mustConfigResourceAddr(fmt.Sprintf("%s%s.bar", prefix, tt.rtype)), + Config: rc, + ResolvedProvider: ResolvedProvider{ProviderConfig: mustProviderConfig(`provider["registry.opentofu.org/hashicorp/aws"]`)}, + }, + } + + ctx := &MockEvalContext{} + ctx.installSimpleEval() + + ctx.ProviderSchemaSchema = mp.GetProviderSchema(t.Context()) + ctx.ProviderProvider = p + + diags := node.validateResource(t.Context(), ctx) + if got, want := len(diags), 1; got != want { + t.Fatalf("expected to have %d diagnostics but got %d", want, got) + } + d := diags[0] + + if got, want := d.Description().Summary, tt.wantSummary; got != want { + t.Fatalf("unexpected diagnostic summary. \nwant:\n\t%s\ngot:\n\t%s", want, got) + } + if got, want := d.Description().Detail, tt.wantDetail; got != want { + t.Fatalf("unexpected diagnostic detail. \nwant:\n\t%s\ngot:\n\t%s", want, got) + } + }) + } +} diff --git a/internal/tofu/node_variable_reference.go b/internal/tofu/node_variable_reference.go index 9d678e33c4..235f22442b 100644 --- a/internal/tofu/node_variable_reference.go +++ b/internal/tofu/node_variable_reference.go @@ -69,6 +69,7 @@ func (n *nodeVariableReference) DynamicExpand(ctx EvalContext) (*Graph, error) { checkableAddrs.Add(addr) } + // TODO ephemeral - variables marks - check on how to ensure ephemeral marks when calling evalVariableValidations func o := &nodeVariableReferenceInstance{ Addr: addr, Config: n.Config, diff --git a/internal/tofu/provider_for_test_framework.go b/internal/tofu/provider_for_test_framework.go index 4e61e0ac38..7131d74206 100644 --- a/internal/tofu/provider_for_test_framework.go +++ b/internal/tofu/provider_for_test_framework.go @@ -113,6 +113,21 @@ func (p providerForTest) ReadDataSource(_ context.Context, r providers.ReadDataS return resp } +func (p providerForTest) OpenEphemeralResource(_ context.Context, _ providers.OpenEphemeralResourceRequest) (resp providers.OpenEphemeralResourceResponse) { + //TODO ephemeral - implement me when adding testing support + panic("implement me") +} + +func (p providerForTest) RenewEphemeralResource(_ context.Context, _ providers.RenewEphemeralResourceRequest) (resp providers.RenewEphemeralResourceResponse) { + //TODO ephemeral - implement me when adding testing support + panic("implement me") +} + +func (p providerForTest) CloseEphemeralResource(_ context.Context, _ providers.CloseEphemeralResourceRequest) (resp providers.CloseEphemeralResourceResponse) { + //TODO ephemeral - implement me when adding testing support + panic("implement me") +} + // ValidateProviderConfig is irrelevant when provider is mocked or overridden. func (p providerForTest) ValidateProviderConfig(_ context.Context, _ providers.ValidateProviderConfigRequest) providers.ValidateProviderConfigResponse { return providers.ValidateProviderConfigResponse{} @@ -160,6 +175,10 @@ func (p providerForTest) UpgradeResourceState(ctx context.Context, r providers.U return p.internal.UpgradeResourceState(ctx, r) } +func (p providerForTest) ValidateEphemeralConfig(ctx context.Context, request providers.ValidateEphemeralConfigRequest) providers.ValidateEphemeralConfigResponse { + return p.internal.ValidateEphemeralConfig(ctx, request) +} + func (p providerForTest) Stop(ctx context.Context) error { return p.internal.Stop(ctx) } diff --git a/internal/tofu/provider_mock.go b/internal/tofu/provider_mock.go index fe777b0aa2..aabffe39cb 100644 --- a/internal/tofu/provider_mock.go +++ b/internal/tofu/provider_mock.go @@ -49,6 +49,12 @@ type MockProvider struct { ValidateDataResourceConfigRequest providers.ValidateDataResourceConfigRequest ValidateDataResourceConfigFn func(providers.ValidateDataResourceConfigRequest) providers.ValidateDataResourceConfigResponse + ValidateEphemeralConfigCalled bool + ValidateEphemeralConfigTypeName string + ValidateEphemeralConfigResponse *providers.ValidateEphemeralConfigResponse + ValidateEphemeralConfigRequest providers.ValidateEphemeralConfigRequest + ValidateEphemeralConfigFn func(providers.ValidateEphemeralConfigRequest) providers.ValidateEphemeralConfigResponse + UpgradeResourceStateCalled bool UpgradeResourceStateTypeName string UpgradeResourceStateResponse *providers.UpgradeResourceStateResponse @@ -95,6 +101,21 @@ type MockProvider struct { ReadDataSourceRequest providers.ReadDataSourceRequest ReadDataSourceFn func(providers.ReadDataSourceRequest) providers.ReadDataSourceResponse + OpenEphemeralResourceCalled bool + OpenEphemeralResourceResponse *providers.OpenEphemeralResourceResponse + OpenEphemeralResourceRequest providers.OpenEphemeralResourceRequest + OpenEphemeralResourceFn func(providers.OpenEphemeralResourceRequest) providers.OpenEphemeralResourceResponse + + RenewEphemeralResourceCalled bool + RenewEphemeralResourceResponse *providers.RenewEphemeralResourceResponse + RenewEphemeralResourceRequest providers.RenewEphemeralResourceRequest + RenewEphemeralResourceFn func(providers.RenewEphemeralResourceRequest) providers.RenewEphemeralResourceResponse + + CloseEphemeralResourceCalled bool + CloseEphemeralResourceResponse *providers.CloseEphemeralResourceResponse + CloseEphemeralResourceRequest providers.CloseEphemeralResourceRequest + CloseEphemeralResourceFn func(providers.CloseEphemeralResourceRequest) providers.CloseEphemeralResourceResponse + GetFunctionsCalled bool GetFunctionsResponse *providers.GetFunctionsResponse GetFunctionsFn func() providers.GetFunctionsResponse @@ -125,9 +146,10 @@ func (p *MockProvider) getProviderSchema() providers.GetProviderSchemaResponse { } return providers.GetProviderSchemaResponse{ - Provider: providers.Schema{}, - DataSources: map[string]providers.Schema{}, - ResourceTypes: map[string]providers.Schema{}, + Provider: providers.Schema{}, + DataSources: map[string]providers.Schema{}, + ResourceTypes: map[string]providers.Schema{}, + EphemeralResources: map[string]providers.Schema{}, } } @@ -214,6 +236,37 @@ func (p *MockProvider) ValidateDataResourceConfig(ctx context.Context, r provide return resp } +func (p *MockProvider) ValidateEphemeralConfig(ctx context.Context, r providers.ValidateEphemeralConfigRequest) (resp providers.ValidateEphemeralConfigResponse) { + tracing.ContextProbeReport(ctx, 0) + p.Lock() + defer p.Unlock() + + p.ValidateEphemeralConfigCalled = true + p.ValidateEphemeralConfigRequest = r + + // Marshall the value to replicate behavior by the GRPC protocol + ephemeralResourceSchema, ok := p.getProviderSchema().EphemeralResources[r.TypeName] + if !ok { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("no schema found for ephemeral resource %q", r.TypeName)) + return resp + } + _, err := msgpack.Marshal(r.Config, ephemeralResourceSchema.Block.ImpliedType()) + if err != nil { + resp.Diagnostics = resp.Diagnostics.Append(err) + return resp + } + + if p.ValidateEphemeralConfigFn != nil { + return p.ValidateEphemeralConfigFn(r) + } + + if p.ValidateEphemeralConfigResponse != nil { + return *p.ValidateEphemeralConfigResponse + } + + return resp +} + func (p *MockProvider) UpgradeResourceState(ctx context.Context, r providers.UpgradeResourceStateRequest) (resp providers.UpgradeResourceStateResponse) { tracing.ContextProbeReport(ctx, 0) p.Lock() @@ -599,6 +652,78 @@ func (p *MockProvider) ReadDataSource(ctx context.Context, r providers.ReadDataS return resp } +func (p *MockProvider) OpenEphemeralResource(ctx context.Context, r providers.OpenEphemeralResourceRequest) (resp providers.OpenEphemeralResourceResponse) { + tracing.ContextProbeReport(ctx, 0) + p.Lock() + defer p.Unlock() + + if !p.ConfigureProviderCalled { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("configure not called before OpenEphemeralResource %q", r.TypeName)) + return resp + } + + p.OpenEphemeralResourceCalled = true + p.OpenEphemeralResourceRequest = r + + if p.OpenEphemeralResourceFn != nil { + return p.OpenEphemeralResourceFn(r) + } + + if p.OpenEphemeralResourceResponse != nil { + resp = *p.OpenEphemeralResourceResponse + } + + return resp +} + +func (p *MockProvider) RenewEphemeralResource(ctx context.Context, r providers.RenewEphemeralResourceRequest) (resp providers.RenewEphemeralResourceResponse) { + tracing.ContextProbeReport(ctx, 0) + p.Lock() + defer p.Unlock() + + if !p.ConfigureProviderCalled { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("configure not called before RenewEphemeralResource %q", r.TypeName)) + return resp + } + + p.RenewEphemeralResourceCalled = true + p.RenewEphemeralResourceRequest = r + + if p.RenewEphemeralResourceFn != nil { + return p.RenewEphemeralResourceFn(r) + } + + if p.RenewEphemeralResourceResponse != nil { + resp = *p.RenewEphemeralResourceResponse + } + + return resp +} + +func (p *MockProvider) CloseEphemeralResource(ctx context.Context, r providers.CloseEphemeralResourceRequest) (resp providers.CloseEphemeralResourceResponse) { + tracing.ContextProbeReport(ctx, 0) + p.Lock() + defer p.Unlock() + + if !p.ConfigureProviderCalled { + resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("configure not called before CloseEphemeralResource %q", r.TypeName)) + return resp + } + + p.CloseEphemeralResourceCalled = true + p.CloseEphemeralResourceRequest = r + + if p.CloseEphemeralResourceFn != nil { + return p.CloseEphemeralResourceFn(r) + } + + if p.CloseEphemeralResourceResponse != nil { + resp = *p.CloseEphemeralResourceResponse + } + + return resp +} + func (p *MockProvider) GetFunctions(ctx context.Context) (resp providers.GetFunctionsResponse) { tracing.ContextProbeReport(ctx, 0) p.Lock() diff --git a/internal/tofu/resource_provider_mock_test.go b/internal/tofu/resource_provider_mock_test.go index adb5fecada..6ddb2b07a6 100644 --- a/internal/tofu/resource_provider_mock_test.go +++ b/internal/tofu/resource_provider_mock_test.go @@ -73,10 +73,13 @@ func simpleMockProvider() *MockProvider { GetProviderSchemaResponse: &providers.GetProviderSchemaResponse{ Provider: providers.Schema{Block: simpleTestSchema()}, ResourceTypes: map[string]providers.Schema{ - "test_object": providers.Schema{Block: simpleTestSchema()}, + "test_object": {Block: simpleTestSchema()}, }, DataSources: map[string]providers.Schema{ - "test_object": providers.Schema{Block: simpleTestSchema()}, + "test_object": {Block: simpleTestSchema()}, + }, + EphemeralResources: map[string]providers.Schema{ + "test_object": {Block: simpleTestSchema()}, }, }, } @@ -92,6 +95,7 @@ func (p *MockProvider) ProviderSchema() *ProviderSchema { ProviderMeta: resp.ProviderMeta.Block, ResourceTypes: map[string]*configschema.Block{}, DataSources: map[string]*configschema.Block{}, + EphemeralTypes: map[string]*configschema.Block{}, ResourceTypeSchemaVersions: map[string]uint64{}, } @@ -104,6 +108,10 @@ func (p *MockProvider) ProviderSchema() *ProviderSchema { schema.DataSources[dataSource] = s.Block } + for ephemeral, s := range resp.EphemeralResources { + schema.EphemeralTypes[ephemeral] = s.Block + } + return schema } @@ -115,16 +123,18 @@ type ProviderSchema struct { ResourceTypes map[string]*configschema.Block ResourceTypeSchemaVersions map[string]uint64 DataSources map[string]*configschema.Block + EphemeralTypes map[string]*configschema.Block } // getProviderSchemaResponseFromProviderSchema is a test helper to convert a // ProviderSchema to a GetProviderSchemaResponse for use when building a mock provider. func getProviderSchemaResponseFromProviderSchema(providerSchema *ProviderSchema) *providers.GetProviderSchemaResponse { resp := &providers.GetProviderSchemaResponse{ - Provider: providers.Schema{Block: providerSchema.Provider}, - ProviderMeta: providers.Schema{Block: providerSchema.ProviderMeta}, - ResourceTypes: map[string]providers.Schema{}, - DataSources: map[string]providers.Schema{}, + Provider: providers.Schema{Block: providerSchema.Provider}, + ProviderMeta: providers.Schema{Block: providerSchema.ProviderMeta}, + ResourceTypes: map[string]providers.Schema{}, + DataSources: map[string]providers.Schema{}, + EphemeralResources: map[string]providers.Schema{}, } for name, schema := range providerSchema.ResourceTypes { @@ -138,5 +148,9 @@ func getProviderSchemaResponseFromProviderSchema(providerSchema *ProviderSchema) resp.DataSources[name] = providers.Schema{Block: schema} } + for name, schema := range providerSchema.EphemeralTypes { + resp.EphemeralResources[name] = providers.Schema{Block: schema} + } + return resp } diff --git a/internal/tofu/schemas_test.go b/internal/tofu/schemas_test.go index ad24337592..cc035cc35c 100644 --- a/internal/tofu/schemas_test.go +++ b/internal/tofu/schemas_test.go @@ -42,6 +42,10 @@ func schemaOnlyProvidersForTesting(schemas map[addrs.Provider]providers.Provider for providerAddr, schema := range schemas { schema := schema + // mark ephemeral resources blocks accordingly + for _, s := range schema.EphemeralResources { + s.Block.Ephemeral = true + } provider := &MockProvider{ GetProviderSchemaResponse: &schema, } diff --git a/internal/tofu/testdata/input-provider/main.tf b/internal/tofu/testdata/input-provider/main.tf index 919f140bba..cf29fa4c8f 100644 --- a/internal/tofu/testdata/input-provider/main.tf +++ b/internal/tofu/testdata/input-provider/main.tf @@ -1 +1,3 @@ resource "aws_instance" "foo" {} +data "cloudflare_account" "bar" {} +ephemeral "azurerm_key_vault_secret" "baz" {} diff --git a/internal/tofu/testdata/static-validate-refs/static-validate-refs.tf b/internal/tofu/testdata/static-validate-refs/static-validate-refs.tf index 2f71e21713..119a722b45 100644 --- a/internal/tofu/testdata/static-validate-refs/static-validate-refs.tf +++ b/internal/tofu/testdata/static-validate-refs/static-validate-refs.tf @@ -22,6 +22,9 @@ resource "boop_whatever" "nope" { data "beep" "boop" { } +ephemeral "foo" "bar" { +} + check "foo" { data "boop_data" "boop_nested" {} diff --git a/internal/tofu/testdata/transform-config-mode-data/main.tf b/internal/tofu/testdata/transform-config-mode-data/main.tf index 3c3e7e50d5..61b2e6c1df 100644 --- a/internal/tofu/testdata/transform-config-mode-data/main.tf +++ b/internal/tofu/testdata/transform-config-mode-data/main.tf @@ -1,3 +1,5 @@ data "aws_ami" "foo" {} resource "aws_instance" "web" {} + +ephemeral "aws_secret" "secret" {} \ No newline at end of file diff --git a/internal/tofu/transform_attach_config_resource.go b/internal/tofu/transform_attach_config_resource.go index 278edd9d4f..e1831edf0f 100644 --- a/internal/tofu/transform_attach_config_resource.go +++ b/internal/tofu/transform_attach_config_resource.go @@ -58,6 +58,8 @@ func (t *AttachResourceConfigTransformer) Transform(_ context.Context, g *Graph) m = config.Module.ManagedResources case addrs.DataResourceMode: m = config.Module.DataResources + case addrs.EphemeralResourceMode: + m = config.Module.EphemeralResources default: panic("unknown resource mode: " + addr.Resource.Mode.String()) } diff --git a/internal/tofu/transform_closeable_resource.go b/internal/tofu/transform_closeable_resource.go new file mode 100644 index 0000000000..0b2a50e6d0 --- /dev/null +++ b/internal/tofu/transform_closeable_resource.go @@ -0,0 +1,129 @@ +// Copyright (c) The OpenTofu Authors +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2023 HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package tofu + +import ( + "context" + + "github.com/opentofu/opentofu/internal/addrs" + "github.com/opentofu/opentofu/internal/dag" + "github.com/opentofu/opentofu/internal/tfdiags" +) + +// resourceCloser is the definition of the function that needs to exist on the node that wants to release external +// resources before exit of the OpenTofu process. +type resourceCloser func() tfdiags.Diagnostics + +// closableResource needs to be implemented by whatever resource wants to be closed before closing the associated +// provider that is handling that type. +type closableResource interface { + GraphNodeModulePath + ResourceAddr() addrs.ConfigResource + Name() string + Close() tfdiags.Diagnostics +} + +// CloseableResourceTransformer is adding new nodes into the graph, responsible with closing specific resource types. +// Right now, there is only the ephemeral resources that are requiring such an action. +type CloseableResourceTransformer struct { + skip bool +} + +func (t *CloseableResourceTransformer) Transform(_ context.Context, g *Graph) error { + if t.skip { + return nil + } + closeableVertices := closeableResourcesVertexMap(g) + cpm := make(map[string]struct{}) + + for key, instances := range closeableVertices { + // check if we already generated a closing node for that particular resource + _, exists := cpm[key] + if exists { + continue + } + + // gather the references for the closing function from all existing instances of the resource + var callbacks []resourceCloser + var addr addrs.ConfigResource + var allDeps []dag.Vertex + for _, inst := range instances { + deps, err := t.collectInstanceDependencies(g, inst) + if err != nil { + return err + } + // collect all vertices inst is linked to + allDeps = append(allDeps, deps...) + // and collect inst as a dependency too, later it will be linked to the node responsible with closing it + allDeps = append(allDeps, inst) + callbacks = append(callbacks, inst.Close) + // Should be the same for all the instances since closeableResourcesVertexMap is generating the key + // based on each vertex addr. Therefore, no issue here on overwriting it on each iteration. + addr = inst.ResourceAddr() + } + // we postponed creation of the node and its addition to the graph, just to ensure that we are having all the + // required information prepared without errors before adding this node into the graph. + ncr := &nodeCloseableResource{ + Addr: addr, + cbs: callbacks, + } + g.Add(ncr) + for _, dep := range allDeps { + g.Connect(dag.BasicEdge(ncr, dep)) + } + cpm[key] = struct{}{} + } + return nil +} + +// collectInstanceDependencies gathers all dependencies of the given node for being linked to the closing node of inst. +// +// To do so, we need to gather two types of dependencies: +// - we need to gather all the provider node of inst. This is needed to ensure that the resource closing +// node will be added as a dependency later to the closing node of the provider, to force the right +// order of nodes execution. +// - gather all the nodes dependent on inst to ensure that we are not executing the closing of the +// resource before having all the other references satisfied. +func (t *CloseableResourceTransformer) collectInstanceDependencies(g *Graph, inst closableResource) ([]dag.Vertex, error) { + var deps []dag.Vertex + + for _, dn := range g.DownEdges(inst) { + switch dn.(type) { + case GraphNodeProvider: + deps = append(deps, dn) + } + } + + desc, err := g.Descendents(inst) + if err != nil { + return nil, err + } + for _, s := range desc { + switch s.(type) { + case GraphNodeReferencer: + deps = append(deps, s) + } + } + + return deps, nil +} + +// closeableResourcesVertexMap collects the vertices that are closableResource and represent an addrs.EphemeralResourceMode. +func closeableResourcesVertexMap(g *Graph) map[string][]closableResource { + m := make(map[string][]closableResource) + for _, v := range g.Vertices() { + if n, ok := v.(closableResource); ok { + addr := n.ResourceAddr() + // Only ephemeral resources are closable for the moment, so ignore anything else + if addr.Resource.Mode != addrs.EphemeralResourceMode { + continue + } + l := m[addr.String()] + m[addr.String()] = append(l, n) + } + } + return m +} diff --git a/internal/tofu/transform_config.go b/internal/tofu/transform_config.go index 7c00068c76..3bdc905903 100644 --- a/internal/tofu/transform_config.go +++ b/internal/tofu/transform_config.go @@ -31,12 +31,12 @@ type ConfigTransformer struct { // Module is the module to add resources from. Config *configs.Config - // Mode will only add resources that match the given mode - ModeFilter bool - Mode addrs.ResourceMode - - // Do not apply this transformer. - skip bool + // ModeFilter can be used choose what resource types to skip from being + // added into the graph from the configuration. + // When this function is not defined, all the resources are allowed. + // When this function is defined, the transformer will add only the + // resources that this function returns "true" on. + ModeFilter func(mode addrs.ResourceMode) bool // importTargets specifies a slice of addresses that will have state // imported for them. @@ -53,10 +53,6 @@ type ConfigTransformer struct { } func (t *ConfigTransformer) Transform(_ context.Context, g *Graph) error { - if t.skip { - return nil - } - // If no configuration is available, we don't do anything if t.Config == nil { return nil @@ -98,13 +94,16 @@ func (t *ConfigTransformer) transformSingle(g *Graph, config *configs.Config, ge module := config.Module log.Printf("[TRACE] ConfigTransformer: Starting for path: %v", path) - allResources := make([]*configs.Resource, 0, len(module.ManagedResources)+len(module.DataResources)) + allResources := make([]*configs.Resource, 0, len(module.ManagedResources)+len(module.DataResources)+len(module.EphemeralResources)) for _, r := range module.ManagedResources { allResources = append(allResources, r) } for _, r := range module.DataResources { allResources = append(allResources, r) } + for _, r := range module.EphemeralResources { + allResources = append(allResources, r) + } // Take a copy of the import targets, so we can edit them as we go. // Only include import targets that are targeting the current module. @@ -118,8 +117,9 @@ func (t *ConfigTransformer) transformSingle(g *Graph, config *configs.Config, ge for _, r := range allResources { relAddr := r.Addr() - if t.ModeFilter && relAddr.Mode != t.Mode { + if t.ModeFilter != nil && t.ModeFilter(relAddr.Mode) { // Skip non-matching modes + log.Printf("[TRACE] config transformer skipped resource %q", relAddr) continue } @@ -174,6 +174,11 @@ func (t *ConfigTransformer) transformSingle(g *Graph, config *configs.Config, ge // We'll add the nodes that we know will fail, and catch them again later // in the processing when we are in a position to raise a much more helpful // error message. + // + // We checked this during the removal of the "skip" argument. + // On walkPlanDestroy, the importTargets it's not even passed across to the plan graph builder. + // Therefore, this is having no impact on the actual behavior of the destroy planning process, + // so we decided not to add additional logic to skip this part. for _, i := range importTargets { if len(generateConfigPath) > 0 { // Create a node with the resource and import target. This node will take care of the config generation diff --git a/internal/tofu/transform_config_test.go b/internal/tofu/transform_config_test.go index cf1c8fef76..184daf48d4 100644 --- a/internal/tofu/transform_config_test.go +++ b/internal/tofu/transform_config_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" "github.com/opentofu/opentofu/internal/addrs" ) @@ -39,22 +40,60 @@ func TestConfigTransformer(t *testing.T) { } func TestConfigTransformer_mode(t *testing.T) { - g := Graph{Path: addrs.RootModuleInstance} - tf := &ConfigTransformer{ - Config: testModule(t, "transform-config-mode-data"), - ModeFilter: true, - Mode: addrs.DataResourceMode, - } - if err := tf.Transform(t.Context(), &g); err != nil { - t.Fatalf("err: %s", err) - } - - actual := strings.TrimSpace(g.String()) - expected := strings.TrimSpace(` + cases := map[string]struct { + filterFunc func(mode addrs.ResourceMode) bool + expectedGraph string + }{ + "no filter": { + filterFunc: nil, + expectedGraph: `aws_instance.web data.aws_ami.foo -`) - if actual != expected { - t.Fatalf("bad:\n\n%s", actual) +ephemeral.aws_secret.secret`, + }, + "allow all": { + filterFunc: func(mode addrs.ResourceMode) bool { + return false + }, + expectedGraph: `aws_instance.web +data.aws_ami.foo +ephemeral.aws_secret.secret`, + }, + "only managed resources": { + filterFunc: func(mode addrs.ResourceMode) bool { + return mode != addrs.ManagedResourceMode + }, + expectedGraph: `aws_instance.web`, + }, + "only data sources": { + filterFunc: func(mode addrs.ResourceMode) bool { + return mode != addrs.DataResourceMode + }, + expectedGraph: `data.aws_ami.foo`, + }, + "only ephemeral resources": { + filterFunc: func(mode addrs.ResourceMode) bool { + return mode != addrs.EphemeralResourceMode + }, + expectedGraph: `ephemeral.aws_secret.secret`, + }, + } + for name, tt := range cases { + t.Run(name, func(t *testing.T) { + g := Graph{Path: addrs.RootModuleInstance} + tf := &ConfigTransformer{ + Config: testModule(t, "transform-config-mode-data"), + ModeFilter: tt.filterFunc, + } + if err := tf.Transform(t.Context(), &g); err != nil { + t.Fatalf("err: %s", err) + } + + actual := strings.TrimSpace(g.String()) + expected := strings.TrimSpace(tt.expectedGraph) + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("wrong graph:\n\n%s", diff) + } + }) } } diff --git a/internal/tofu/transform_destroy_edge.go b/internal/tofu/transform_destroy_edge.go index 3a1a4fd2ff..578520715b 100644 --- a/internal/tofu/transform_destroy_edge.go +++ b/internal/tofu/transform_destroy_edge.go @@ -174,9 +174,15 @@ func (t *DestroyEdgeTransformer) Transform(_ context.Context, g *Graph) error { break } - // NoOp changes should not participate in the destroy dependencies. + // NoOp and Open changes should not participate in the destroy dependencies. + // + // The Open changes have been added later, with the introduction of ephemeral resources. + // The idea is that ephemeral resources cannot be dependent on destroying resources since + // the best case scenario, the ephemeral resource can only provide some information to + // another dependency of the resource that it's going to be destroyed, but that is done + // through other transformers, as ReferenceTransformer. rc := t.Changes.ResourceInstance(*addr) - if rc != nil && rc.Action != plans.NoOp { + if rc != nil && rc.Action != plans.NoOp && rc.Action != plans.Open { creators[cfgAddr] = append(creators[cfgAddr], n) } } @@ -309,6 +315,7 @@ func (t *pruneUnusedNodesTransformer) Transform(_ context.Context, g *Graph) err // root module outputs indicate they are not temporary by // returning false here. if !n.temporaryValue() { + log.Printf("[TRACE] pruneUnusedNodes: temporary value vertex %q kept because it's not a temporary value vertex", dag.VertexName(n)) return } @@ -318,6 +325,7 @@ func (t *pruneUnusedNodesTransformer) Transform(_ context.Context, g *Graph) err // keep any value which is connected through a // reference if _, ok := v.(GraphNodeReferencer); ok { + log.Printf("[TRACE] pruneUnusedNodes: temporary value vertex %q kept it is referenced by %q", dag.VertexName(n), dag.VertexName(v)) return } } @@ -335,15 +343,23 @@ func (t *pruneUnusedNodesTransformer) Transform(_ context.Context, g *Graph) err // to expand it and so this lets us do a bit more // pruning than we'd be able to do otherwise. if tmp, ok := v.(graphNodeTemporaryValue); ok && !tmp.temporaryValue() { + log.Printf("[TRACE] pruneUnusedNodes: expanding vertex %q kept because another expanding vertex %q with non-temporary value is one of its dependencies", dag.VertexName(n), dag.VertexName(v)) continue } // expanders can always depend on module expansion // themselves + log.Printf("[TRACE] pruneUnusedNodes: expanding vertex %q kept because another expanding vertex %q is one of its dependencies", dag.VertexName(n), dag.VertexName(v)) return case GraphNodeResourceInstance: // resource instances always depend on their // resource node, which is an expander + log.Printf("[TRACE] pruneUnusedNodes: expanding vertex %q kept because an instance vertex %q depends on it", dag.VertexName(n), dag.VertexName(v)) + return + case GraphNodeProvider: + // When a provider is referencing a resource managed by a different provider instance, + // it means that we need to run that resource before actually configuring the dependant provider. + log.Printf("[TRACE] pruneUnusedNodes: expanding vertex %q kept because a provider vertex %q depends on it", dag.VertexName(n), dag.VertexName(v)) return } } @@ -361,13 +377,16 @@ func (t *pruneUnusedNodesTransformer) Transform(_ context.Context, g *Graph) err for _, v := range des { switch v.(type) { case GraphNodeProviderConsumer: + log.Printf("[TRACE] pruneUnusedNodes: provider vertex %q kept because vertex %q depends on it", dag.VertexName(n), dag.VertexName(v)) return case GraphNodeReferencer: + log.Printf("[TRACE] pruneUnusedNodes: provider vertex %q kept because vertex %q is referencing it", dag.VertexName(n), dag.VertexName(v)) return } } default: + log.Printf("[TRACE] pruneUnusedNodes: vertex %q kept", dag.VertexName(n)) return } diff --git a/internal/tofu/transform_destroy_edge_test.go b/internal/tofu/transform_destroy_edge_test.go index d99861bd9b..3ae3bebdd5 100644 --- a/internal/tofu/transform_destroy_edge_test.go +++ b/internal/tofu/transform_destroy_edge_test.go @@ -459,12 +459,13 @@ func TestPruneUnusedNodesTransformer_rootModuleOutputValues(t *testing.T) { } } -// NoOp changes should not be participating in the destroy sequence +// NoOp and Open changes should not be participating in the destroy sequence func TestDestroyEdgeTransformer_noOp(t *testing.T) { g := Graph{Path: addrs.RootModuleInstance} g.Add(testDestroyNode("test_object.A")) g.Add(testUpdateNode("test_object.B")) g.Add(testDestroyNode("test_object.C")) + g.Add(testUpdateNode("ephemeral.test_object.D")) state := states.NewState() root := state.EnsureModule(addrs.RootModuleInstance) @@ -480,9 +481,12 @@ func TestDestroyEdgeTransformer_noOp(t *testing.T) { root.SetResourceInstanceCurrent( mustResourceInstanceAddr("test_object.B").Resource, &states.ResourceInstanceObjectSrc{ - Status: states.ObjectReady, - AttrsJSON: []byte(`{"id":"B","test_string":"x"}`), - Dependencies: []addrs.ConfigResource{mustConfigResourceAddr("test_object.A")}, + Status: states.ObjectReady, + AttrsJSON: []byte(`{"id":"B","test_string":"x"}`), + Dependencies: []addrs.ConfigResource{ + mustConfigResourceAddr("test_object.A"), + mustConfigResourceAddr("ephemeral.test_object.D"), + }, }, mustProviderConfig(`provider["registry.opentofu.org/hashicorp/test"]`), addrs.NoKey, @@ -492,8 +496,11 @@ func TestDestroyEdgeTransformer_noOp(t *testing.T) { &states.ResourceInstanceObjectSrc{ Status: states.ObjectReady, AttrsJSON: []byte(`{"id":"C","test_string":"x"}`), - Dependencies: []addrs.ConfigResource{mustConfigResourceAddr("test_object.A"), - mustConfigResourceAddr("test_object.B")}, + Dependencies: []addrs.ConfigResource{ + mustConfigResourceAddr("test_object.A"), + mustConfigResourceAddr("test_object.B"), + mustConfigResourceAddr("ephemeral.test_object.D"), + }, }, mustProviderConfig(`provider["registry.opentofu.org/hashicorp/test"]`), addrs.NoKey, @@ -504,14 +511,20 @@ func TestDestroyEdgeTransformer_noOp(t *testing.T) { } tf := &DestroyEdgeTransformer{ - // We only need a minimal object to indicate GraphNodeCreator change is - // a NoOp here. Changes: &plans.Changes{ Resources: []*plans.ResourceInstanceChangeSrc{ + // We only need a minimal object to indicate GraphNodeCreator change is + // a NoOp here. { Addr: mustResourceInstanceAddr("test_object.B"), ChangeSrc: plans.ChangeSrc{Action: plans.NoOp}, }, + // We only need a minimal object to indicate GraphNodeCreator change is + // an Open here. + { + Addr: mustResourceInstanceAddr("ephemeral.test_object.D"), + ChangeSrc: plans.ChangeSrc{Action: plans.Open}, + }, }, }, } @@ -520,6 +533,7 @@ func TestDestroyEdgeTransformer_noOp(t *testing.T) { } expected := strings.TrimSpace(` +ephemeral.test_object.D test_object.A (destroy) test_object.C (destroy) test_object.B diff --git a/internal/tofu/transform_diff_test.go b/internal/tofu/transform_diff_test.go index f49a90284e..6c35ca0a68 100644 --- a/internal/tofu/transform_diff_test.go +++ b/internal/tofu/transform_diff_test.go @@ -14,6 +14,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/opentofu/opentofu/internal/configs" "github.com/opentofu/opentofu/internal/refactoring" "github.com/zclconf/go-cty/cty" @@ -221,3 +222,74 @@ func TestTransformRemovedProvisioners(t *testing.T) { t.Fatalf("expected no diff between the expected provisioners and the ones configured in NodeDestroyResourceInstance. got:\n %s", diff) } } + +// TestDiffTransformerEphemeralChanges is having some wierd and theoretically impossible setup steps, but it's done +// this way to verify that some checks are in place along the way. Check the inline comments for more details. +func TestDiffTransformerEphemeralChanges(t *testing.T) { + g := Graph{Path: addrs.RootModuleInstance} + + beforeVal, err := plans.NewDynamicValue(cty.StringVal(""), cty.String) + if err != nil { + t.Fatal(err) + } + afterVal, err := plans.NewDynamicValue(cty.StringVal(""), cty.String) + if err != nil { + t.Fatal(err) + } + tf := &DiffTransformer{ + Config: &configs.Config{ + Module: &configs.Module{}, + }, + Changes: &plans.Changes{ + Resources: []*plans.ResourceInstanceChangeSrc{ + { + Addr: addrs.Resource{ + // This is the wierd/stupid setup: the list of changes is NEVER meant to contain an ephemeral resource + // since those are computed from the state compared with the currently wanted configuration. + // Since ephemeral resources are not meant to be stored in the state file, this should never happen. + // This setup is done this way only to be sure that the code path for creating NodeDestroyResourceInstance + // is working well and that the node resulted from that returns an error on v.Execute(...) + Mode: addrs.EphemeralResourceMode, + Type: "aws_secretmanager_secret", + Name: "foo", + }.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance), + ProviderAddr: addrs.AbsProviderConfig{ + Provider: addrs.NewDefaultProvider("aws"), + Module: addrs.RootModule, + }, + ChangeSrc: plans.ChangeSrc{ + Action: plans.Delete, + Before: beforeVal, + After: afterVal, + }, + }, + }, + }, + } + if err := tf.Transform(t.Context(), &g); err != nil { + t.Fatalf("err: %s", err) + } + + actual := strings.TrimSpace(g.String()) + expected := "ephemeral.aws_secretmanager_secret.foo (destroy)" + if actual != expected { + t.Fatalf("bad:\n\n%s", actual) + } + + verts := g.Vertices() + if got := len(verts); got != 1 { + t.Fatalf("expected to have exactly one vertex. got: %d", got) + } + + // Let's see how NodeDestroyResourceInstance.Execute is behaving when it's for an ephemeral resource + v, ok := verts[0].(*NodeDestroyResourceInstance) + if !ok { + t.Fatalf("expected that the only created vertex to be NodeDestroyResourceInstance. got %s", reflect.TypeOf(verts[0])) + } + diags := v.Execute(t.Context(), nil, walkApply) + got := diags.Err().Error() + want := "Destroy invoked for an ephemeral resource: A destroy operation has been invoked for the ephemeral resource \"ephemeral.aws_secretmanager_secret.foo\". This is an OpenTofu error. Please report this." + if got != want { + t.Fatalf("unexpected error returned.\ngot: %s\nwant:%s", got, want) + } +} diff --git a/internal/tofu/transform_provider.go b/internal/tofu/transform_provider.go index 8bba815900..dfdf41962c 100644 --- a/internal/tofu/transform_provider.go +++ b/internal/tofu/transform_provider.go @@ -457,6 +457,10 @@ func (t *CloseProviderTransformer) Transform(_ context.Context, g *Graph) error g.Connect(dag.BasicEdge(closer, s)) } else if _, ok := s.(GraphNodeReferencer); ok { g.Connect(dag.BasicEdge(closer, s)) + } else if _, ok := s.(GraphNodeCloseableResource); ok { + // Connect also the nodes that are meant to close resources since these + // are created quite late in the graph building process + g.Connect(dag.BasicEdge(closer, s)) } } } diff --git a/internal/tofu/transform_reference.go b/internal/tofu/transform_reference.go index ef5be9eaf9..4c35eaa564 100644 --- a/internal/tofu/transform_reference.go +++ b/internal/tofu/transform_reference.go @@ -65,28 +65,28 @@ type graphNodeDependsOn interface { DependsOn() []*addrs.Reference } -// graphNodeAttachDataResourceDependsOn records all resources that are transitively +// graphNodeAttachResourceDependsOn records all resources that are transitively // referenced through depends_on in the configuration. This is used by data -// resources to determine if they can be read during the plan, or if they need -// to be further delayed until apply. +// resources and ephemeral resources to determine if they can be processed during the plan, +// or if they need to be further delayed until apply. // We can only use an addrs.ConfigResource address here, because modules are // not yet expended in the graph. While this will cause some extra data // resources to show in the plan when their depends_on references may be in // unrelated module instances, the fact that it only happens when there are any // resource updates pending means we can still avoid the problem of the // "perpetual diff" -type graphNodeAttachDataResourceDependsOn interface { +type graphNodeAttachResourceDependsOn interface { GraphNodeConfigResource graphNodeDependsOn - // AttachDataResourceDependsOn stores the discovered dependencies in the + // AttachResourceDependsOn stores the discovered dependencies in the // resource node for evaluation later. // // The force parameter indicates that even if there are no dependencies, // force the data source to act as though there are for refresh purposes. // This is needed because yet-to-be-created resources won't be in the // initial refresh graph, but may still be referenced through depends_on. - AttachDataResourceDependsOn(deps []addrs.ConfigResource, force bool) + AttachResourceDependsOn(deps []addrs.ConfigResource, force bool) } // GraphNodeReferenceOutside is an interface that can optionally be implemented. @@ -181,12 +181,13 @@ func (m depMap) add(v dag.Vertex) { } } -// attachDataResourceDependsOnTransformer records all resources transitively +// attachResourceDependsOnTransformer records all resources transitively // referenced through a configuration depends_on. -type attachDataResourceDependsOnTransformer struct { +// This transformer is applicable strictly for data sources and ephemeral resources. +type attachResourceDependsOnTransformer struct { } -func (t attachDataResourceDependsOnTransformer) Transform(_ context.Context, g *Graph) error { +func (t attachResourceDependsOnTransformer) Transform(_ context.Context, g *Graph) error { // First we need to make a map of referenceable addresses to their vertices. // This is very similar to what's done in ReferenceTransformer, but we keep // implementation separate as they may need to change independently. @@ -194,14 +195,14 @@ func (t attachDataResourceDependsOnTransformer) Transform(_ context.Context, g * refMap := NewReferenceMap(vertices) for _, v := range vertices { - depender, ok := v.(graphNodeAttachDataResourceDependsOn) + depender, ok := v.(graphNodeAttachResourceDependsOn) if !ok { continue } - // Only data need to attach depends_on, so they can determine if they + // Only data and ephemeral need to attach depends_on, so they can determine if they // are eligible to be read during plan. - if depender.ResourceAddr().Resource.Mode != addrs.DataResourceMode { + if depender.ResourceAddr().Resource.Mode != addrs.DataResourceMode && depender.ResourceAddr().Resource.Mode != addrs.EphemeralResourceMode { continue } @@ -218,8 +219,8 @@ func (t attachDataResourceDependsOnTransformer) Transform(_ context.Context, g * res = append(res, d) } - log.Printf("[TRACE] attachDataDependenciesTransformer: %s depends on %s", depender.ResourceAddr(), res) - depender.AttachDataResourceDependsOn(res, fromModule) + log.Printf("[TRACE] attachResourceDependsOnTransformer: %s depends on %s", depender.ResourceAddr(), res) + depender.AttachResourceDependsOn(res, fromModule) } return nil @@ -369,8 +370,8 @@ func (m ReferenceMap) dependsOn(g *Graph, depender graphNodeDependsOn) ([]dag.Ve refs := depender.DependsOn() - // get any implied dependencies for data sources - refs = append(refs, m.dataDependsOn(depender)...) + // get any implied dependencies of the depender + refs = append(refs, m.nodeDependencies(depender)...) // This is where we record that a module has depends_on configured. if _, ok := depender.(*nodeExpandModule); ok && len(refs) > 0 { @@ -396,11 +397,11 @@ func (m ReferenceMap) dependsOn(g *Graph, depender graphNodeDependsOn) ([]dag.Ve // Check any ancestors for transitive dependencies when we're // not pointed directly at a resource. We can't be much more - // precise here, since in order to maintain our guarantee that data - // sources will wait for explicit dependencies, if those dependencies - // happen to be a module, output, or variable, we have to find some - // upstream managed resource in order to check for a planned - // change. + // precise here, since in order to maintain our guarantee that + // the depender (data or ephemeral) will wait for explicit dependencies, + // if those dependencies happen to be a module, output, or variable, + // we have to find some upstream managed resource in order to check for + // a planned change. if _, ok := rv.(GraphNodeConfigResource); !ok { ans, _ := g.Ancestors(rv) for _, v := range ans { @@ -418,43 +419,48 @@ func (m ReferenceMap) dependsOn(g *Graph, depender graphNodeDependsOn) ([]dag.Ve return res, fromModule || fromParentModule } -// Return extra depends_on references if this is a data source. -// For data sources we implicitly treat references to managed resources as -// depends_on entries. If a data source references a managed resource, even if +// Return extra depends_on references if "depender" is a data source or an ephemeral resource. +// For the "depender" we implicitly treat references to managed resources as +// depends_on entries. If "depender" references a managed resource, even if // that reference is resolvable, it stands to reason that the user intends for -// the data source to require that resource in some way. -func (m ReferenceMap) dataDependsOn(depender graphNodeDependsOn) []*addrs.Reference { +// the "depender" to require that resource in some way. +func (m ReferenceMap) nodeDependencies(depender graphNodeDependsOn) []*addrs.Reference { var refs []*addrs.Reference - if n, ok := depender.(GraphNodeConfigResource); ok && - n.ResourceAddr().Resource.Mode == addrs.DataResourceMode { - for _, r := range depender.References() { + n, ok := depender.(GraphNodeConfigResource) + if !ok { + return refs + } - var resAddr addrs.Resource - switch s := r.Subject.(type) { - case addrs.Resource: - resAddr = s - case addrs.ResourceInstance: - resAddr = s.Resource - r.Subject = resAddr - case addrs.ProviderFunction: - continue - } + if n.ResourceAddr().Resource.Mode != addrs.DataResourceMode && n.ResourceAddr().Resource.Mode != addrs.EphemeralResourceMode { + return refs + } - if resAddr.Mode != addrs.ManagedResourceMode { - // We only want to wait on directly referenced managed resources. - // Data sources have no external side effects, so normal - // references to them in the config will suffice for proper - // ordering. - continue - } - - refs = append(refs, r) + for _, r := range depender.References() { + var resAddr addrs.Resource + switch s := r.Subject.(type) { + case addrs.Resource: + resAddr = s + case addrs.ResourceInstance: + resAddr = s.Resource + r.Subject = resAddr + case addrs.ProviderFunction: + continue } + + if resAddr.Mode != addrs.ManagedResourceMode { + // We only want to wait on directly referenced managed resources. + // Data sources and ephemeral resources have no external side effects, so normal + // references to them in the config will suffice for proper + // ordering. + continue + } + + refs = append(refs, r) } return refs } -// parentModuleDependsOn returns the set of vertices that a data sources parent +// parentModuleDependsOn returns the state of vertices that a data sources parent // module references through the module call's depends_on. The bool return // value indicates if depends_on was found in a parent module configuration. func (m ReferenceMap) parentModuleDependsOn(g *Graph, depender graphNodeDependsOn) ([]dag.Vertex, bool) { diff --git a/internal/tofu/transform_reference_test.go b/internal/tofu/transform_reference_test.go index 9b86437161..957610fffa 100644 --- a/internal/tofu/transform_reference_test.go +++ b/internal/tofu/transform_reference_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/opentofu/opentofu/internal/addrs" + "github.com/opentofu/opentofu/internal/configs/configschema" "github.com/opentofu/opentofu/internal/dag" ) @@ -177,6 +178,104 @@ module.foo[1].thing.b } } +// TestAttachResourceDependsOnTransformer performs a sanity check on the happy path +// of attachResourceDependsOnTransformer. +// This expects to find the dependency injected into the graph nodes that this +// transformer is working with (data sources and ephemeral resource). +func TestAttachResourceDependsOnTransformer(t *testing.T) { + cfg := testModuleInline(t, map[string]string{ + "main.tf": ` +resource "null_instance" "write" { + foo = "attribute" +} + +data "null_data_source" "read" { + depends_on = ["null_instance.write"] +} + +ephemeral "null_ephemeral" "open" { + depends_on = ["null_instance.write"] +} +`, + }) + g := Graph{Path: addrs.RootModuleInstance} + resCfg := cfg.Module.ResourceByAddr(addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "null_instance", + Name: "write", + }) + resAddr := resCfg.Addr().InModule(addrs.RootModule) + g.Add(&NodeAbstractResourceInstance{ + NodeAbstractResource: NodeAbstractResource{Addr: resAddr, Config: resCfg}, + Addr: addrs.AbsResourceInstance{ + Module: addrs.ModuleInstance{}, + Resource: addrs.ResourceInstance{ + Resource: resCfg.Addr(), + Key: addrs.NoKey, + }, + }}) + + dataCfg := cfg.Module.ResourceByAddr(addrs.Resource{ + Mode: addrs.DataResourceMode, + Type: "null_data_source", + Name: "read", + }) + dataAddr := dataCfg.Addr().InModule(addrs.RootModule) + dataNode := &NodeAbstractResourceInstance{ + NodeAbstractResource: NodeAbstractResource{Addr: dataAddr, Config: dataCfg, + Schema: &configschema.Block{}, // Setting this strictly to force references processing (check GraphNodeReferencer) + }, + Addr: addrs.AbsResourceInstance{ + Module: addrs.ModuleInstance{}, + Resource: addrs.ResourceInstance{ + Resource: dataCfg.Addr(), + Key: addrs.NoKey, + }, + }, + } + g.Add(dataNode) + + ephemeralCfg := cfg.Module.ResourceByAddr(addrs.Resource{ + Mode: addrs.EphemeralResourceMode, + Type: "null_ephemeral", + Name: "open", + }) + ephemeralAddr := ephemeralCfg.Addr().InModule(addrs.RootModule) + ephemeralNode := &NodeAbstractResourceInstance{ + NodeAbstractResource: NodeAbstractResource{Addr: ephemeralAddr, Config: ephemeralCfg, + Schema: &configschema.Block{}, // Setting this strictly to force references processing (check GraphNodeReferencer) + }, + Addr: addrs.AbsResourceInstance{ + Module: addrs.ModuleInstance{}, + Resource: addrs.ResourceInstance{ + Resource: ephemeralCfg.Addr(), + Key: addrs.NoKey, + }, + }, + } + g.Add(ephemeralNode) + + tr := &attachResourceDependsOnTransformer{} + err := tr.Transform(t.Context(), &g) + if err != nil { + t.Fatalf("expected no error. got %s", err) + } + + if got, want := len(dataNode.dependsOn), 1; got != want { + t.Fatalf("wrong number of deps on the data source graph node. expected %d but got %d", want, got) + } + if got, want := dataNode.dependsOn[0], resAddr; !got.Equal(want) { + t.Fatalf("wrong reference registered as dependency. expected %+v but got %+v", want, got) + } + + if got, want := len(ephemeralNode.dependsOn), 1; got != want { + t.Fatalf("wrong number of deps on the data source graph node. expected %d but got %d", want, got) + } + if got, want := ephemeralNode.dependsOn[0], resAddr; !got.Equal(want) { + t.Fatalf("wrong reference registered as dependency. expected %+v but got %+v", want, got) + } +} + func TestReferenceMapReferences(t *testing.T) { cases := map[string]struct { Nodes []dag.Vertex