Update equivalence testing framework to resolve flaky tests (#36179)

* Attempt to remove flaky parts of the equivalence testing

* use new version to fix this

* update with latest terraform
This commit is contained in:
Liam Cervante 2024-12-12 15:54:26 +01:00 committed by GitHub
parent aa38305344
commit 3fd0cc04be
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
47 changed files with 4 additions and 108 deletions

View file

@ -38,7 +38,7 @@ jobs:
shell: bash
run: |
./.github/scripts/equivalence-test.sh download_equivalence_test_binary \
0.4.0 \
0.5.0 \
./bin/equivalence-tests \
linux \
amd64

View file

@ -13,8 +13,8 @@ on:
required: true
equivalence-test-version:
type: string
description: 'Equivalence testing framework version to use (no v prefix, eg. 0.4.0).'
default: "0.4.0"
description: 'Equivalence testing framework version to use (no v prefix, eg. 0.5.0).'
default: "0.5.0"
required: true
permissions:

View file

@ -60,7 +60,7 @@ jobs:
- uses: ./.github/actions/equivalence-test
with:
target-equivalence-test-version: 0.4.0
target-equivalence-test-version: 0.5.0
target-os: linux
target-arch: amd64
current-branch: ${{ github.event.pull_request.base.ref }}

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.json: Modifications complete after 0s [id=5a3fd9b3-e852-8956-8c0a-255d47eda645]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "5a3fd9b3-e852-8956-8c0a-255d47eda645",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_list.list: Creation complete after 0s [id=985820B3-ACF9-4F00-94AD-F81C5EA33663]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "985820B3-ACF9-4F00-94AD-F81C5EA33663",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_list.list: Modifications complete after 0s [id=985820B3-ACF9-4F00-94AD-F81C5EA33663]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "985820B3-ACF9-4F00-94AD-F81C5EA33663",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_list.list: Modifications complete after 0s [id=985820B3-ACF9-4F00-94AD-F81C5EA33663]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "985820B3-ACF9-4F00-94AD-F81C5EA33663",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_list.list: Modifications complete after 0s [id=985820B3-ACF9-4F00-94AD-F81C5EA33663]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "985820B3-ACF9-4F00-94AD-F81C5EA33663",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_map.map: Creation complete after 0s [id=50E1A46E-E64A-4C1F-881C-BA85A5440964]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "50E1A46E-E64A-4C1F-881C-BA85A5440964",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_map.map: Modifications complete after 0s [id=50E1A46E-E64A-4C1F-881C-BA85A5440964]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "50E1A46E-E64A-4C1F-881C-BA85A5440964",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_map.map: Modifications complete after 0s [id=50E1A46E-E64A-4C1F-881C-BA85A5440964]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "50E1A46E-E64A-4C1F-881C-BA85A5440964",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_map.map: Modifications complete after 0s [id=50E1A46E-E64A-4C1F-881C-BA85A5440964]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "50E1A46E-E64A-4C1F-881C-BA85A5440964",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.multiline: Modifications complete after 0s [id=69fe5233-e77a-804f-0dac-115c949540bc]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "69fe5233-e77a-804f-0dac-115c949540bc",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_set.set: Creation complete after 0s [id=046952C9-B832-4106-82C0-C217F7C73E18]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "046952C9-B832-4106-82C0-C217F7C73E18",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_set.set: Modifications complete after 0s [id=046952C9-B832-4106-82C0-C217F7C73E18]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "046952C9-B832-4106-82C0-C217F7C73E18",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_set.set: Modifications complete after 0s [id=046952C9-B832-4106-82C0-C217F7C73E18]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "046952C9-B832-4106-82C0-C217F7C73E18",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_set.set: Modifications complete after 0s [id=046952C9-B832-4106-82C0-C217F7C73E18]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "046952C9-B832-4106-82C0-C217F7C73E18",
"resource": {

View file

@ -57,11 +57,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.base: Modifications complete after 0s [id=f6f74ca6-e8ef-e51f-522c-433b9ed5038f]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "f6f74ca6-e8ef-e51f-522c-433b9ed5038f",
"resource": {
@ -98,11 +96,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.dependent: Modifications complete after 0s [id=1b17b502-96c9-fcc3-3b09-2af1c3de6ad8]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "1b17b502-96c9-fcc3-3b09-2af1c3de6ad8",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.drift: Modifications complete after 0s [id=f3c6ddc5-37d5-0170-64ff-518ad421385a]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "f3c6ddc5-37d5-0170-64ff-518ad421385a",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_complex_resource.complex: Creation complete after 0s [id=64564E36-BFCB-458B-9405-EBBF6A3CAC7A]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "64564E36-BFCB-458B-9405-EBBF6A3CAC7A",
"resource": {

View file

@ -90,11 +90,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_complex_resource.complex: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "tfcoremock_complex_resource.complex",
"implied_provider": "tfcoremock",

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_complex_resource.complex: Modifications complete after 0s [id=64564E36-BFCB-458B-9405-EBBF6A3CAC7A]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "64564E36-BFCB-458B-9405-EBBF6A3CAC7A",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "local_file.local_file: Creation complete after 0s [id=2248ee2fa0aaaad99178531f924bf00b4b0a8f4e]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "2248ee2fa0aaaad99178531f924bf00b4b0a8f4e",
"resource": {

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "local_file.local_file: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "local_file.local_file",
"implied_provider": "local",

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "local_file.local_file: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "local_file.local_file",
"implied_provider": "local",
@ -77,11 +75,9 @@
},
{
"@level": "info",
"@message": "local_file.local_file: Creation complete after 0s [id=648a5452054fca119f95b07f9ea992cc6d9681df]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "648a5452054fca119f95b07f9ea992cc6d9681df",
"resource": {

View file

@ -66,11 +66,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.base_after: Modifications complete after 0s [id=e450ef2f-b80f-0cce-8bdb-14d88f48649c]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "e450ef2f-b80f-0cce-8bdb-14d88f48649c",
"resource": {
@ -107,11 +105,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.dependent: Modifications complete after 0s [id=2ecc718c-8d04-5774-5c36-7d69bf77d34e]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "2ecc718c-8d04-5774-5c36-7d69bf77d34e",
"resource": {

View file

@ -48,11 +48,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_simple_resource.moved: Modifications complete after 0s [id=7da63aeb-f908-a112-9886-f29a0b0bd4ad]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "7da63aeb-f908-a112-9886-f29a0b0bd4ad",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_multiple_blocks.multiple_blocks: Creation complete after 0s [id=DA051126-BAD6-4EB2-92E5-F0250DAF0B92]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "DA051126-BAD6-4EB2-92E5-F0250DAF0B92",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_multiple_blocks.multiple_blocks: Modifications complete after 0s [id=DA051126-BAD6-4EB2-92E5-F0250DAF0B92]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "DA051126-BAD6-4EB2-92E5-F0250DAF0B92",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_list.nested_list: Creation complete after 0s [id=DA051126-BAD6-4EB2-92E5-F0250DAF0B92]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "DA051126-BAD6-4EB2-92E5-F0250DAF0B92",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_list.nested_list: Modifications complete after 0s [id=DA051126-BAD6-4EB2-92E5-F0250DAF0B92]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "DA051126-BAD6-4EB2-92E5-F0250DAF0B92",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_map.nested_map: Creation complete after 0s [id=502B0348-B796-4F6A-8694-A5A397237B85]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "502B0348-B796-4F6A-8694-A5A397237B85",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_map.nested_map: Modifications complete after 0s [id=502B0348-B796-4F6A-8694-A5A397237B85]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "502B0348-B796-4F6A-8694-A5A397237B85",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_object.nested_object: Creation complete after 0s [id=B2491EF0-9361-40FD-B25A-0332A1A5E052]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "B2491EF0-9361-40FD-B25A-0332A1A5E052",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_object.nested_object: Modifications complete after 0s [id=B2491EF0-9361-40FD-B25A-0332A1A5E052]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "B2491EF0-9361-40FD-B25A-0332A1A5E052",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_set.nested_set: Creation complete after 0s [id=510598F6-83FE-4090-8986-793293E90480]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "510598F6-83FE-4090-8986-793293E90480",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_nested_set.nested_set: Modifications complete after 0s [id=510598F6-83FE-4090-8986-793293E90480]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "510598F6-83FE-4090-8986-793293E90480",
"resource": {

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "null_resource.null_resource: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "null_resource.null_resource",
"implied_provider": "null",

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_list.list: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "tfcoremock_list.list",
"implied_provider": "tfcoremock",
@ -77,11 +75,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_list.list: Creation complete after 0s [id=F40F2AB4-100C-4AE8-BFD0-BF332A158415]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "F40F2AB4-100C-4AE8-BFD0-BF332A158415",
"resource": {

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_map.map: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "tfcoremock_map.map",
"implied_provider": "tfcoremock",
@ -77,11 +75,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_map.map: Creation complete after 0s [id=F40F2AB4-100C-4AE8-BFD0-BF332A158415]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "F40F2AB4-100C-4AE8-BFD0-BF332A158415",
"resource": {

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "tfcoremock_object.object",
"implied_provider": "tfcoremock",
@ -77,11 +75,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Creation complete after 0s [id=F40F2AB4-100C-4AE8-BFD0-BF332A158415]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "F40F2AB4-100C-4AE8-BFD0-BF332A158415",
"resource": {

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_set.set: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "tfcoremock_set.set",
"implied_provider": "tfcoremock",
@ -77,11 +75,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_set.set: Creation complete after 0s [id=F40F2AB4-100C-4AE8-BFD0-BF332A158415]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "F40F2AB4-100C-4AE8-BFD0-BF332A158415",
"resource": {

View file

@ -37,11 +37,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Creation complete after 0s [id=AF9833AE-3434-4D0B-8B69-F4B992565D9F]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "AF9833AE-3434-4D0B-8B69-F4B992565D9F",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Modifications complete after 0s [id=00e14fba-4d56-6cc5-b685-633555376e3f]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "00e14fba-4d56-6cc5-b685-633555376e3f",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Modifications complete after 0s [id=00e14fba-4d56-6cc5-b685-633555376e3f]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "00e14fba-4d56-6cc5-b685-633555376e3f",
"resource": {

View file

@ -40,11 +40,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Destruction complete after 0s",
"@module": "terraform.ui",
"hook": {
"action": "delete",
"elapsed_seconds": 0,
"resource": {
"addr": "tfcoremock_object.object",
"implied_provider": "tfcoremock",
@ -77,11 +75,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Creation complete after 0s [id=63A9E8E8-71BC-4DAE-A66C-48CE393CCBD3]",
"@module": "terraform.ui",
"hook": {
"action": "create",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "63A9E8E8-71BC-4DAE-A66C-48CE393CCBD3",
"resource": {

View file

@ -39,11 +39,9 @@
},
{
"@level": "info",
"@message": "tfcoremock_object.object: Modifications complete after 0s [id=00e14fba-4d56-6cc5-b685-633555376e3f]",
"@module": "terraform.ui",
"hook": {
"action": "update",
"elapsed_seconds": 0,
"id_key": "id",
"id_value": "00e14fba-4d56-6cc5-b685-633555376e3f",
"resource": {