Skip to content

Commit 72f22b0

Browse files
cojencocojenco
authored
tests: add retry conformance test cases and markdown (#594)
* add scenario 5 to test schema * align json with conformance test repo * add scenario 6 to test schema * sort import statements * add md file for retry conf tests * revise markdown
1 parent 6085d85 commit 72f22b0

File tree

3 files changed

+149
-9
lines changed

3 files changed

+149
-9
lines changed

‎tests/conformance/retry_conformance_testing.md

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Retry Strategy Conformance Testing
2+
3+
Calls fail for a host of transient reasons. In many cases the failures are ones that should be abstracted from customers, e.g. retryable issues. Retry strategies are included in the client library when it's safe to do so. These retry strategies are complex and need automated tests to ensure they work and continue to work in the future.
4+
5+
The Retry Strategy Conformance tests will ensure that retries are aligned across languages and operations are retried as specified.
6+
7+
## Test Suite Overview
8+
9+
The Retry Strategy Conformance tests leverage the conformance tests defined in [googleapis/conformance-tests](https://github.com/googleapis/conformance-tests/blob/master/storage/v1/retry_tests.json) to ensure adherence to expected behaviors.
10+
11+
The test suite uses the [storage-testbench](https://github.com/googleapis/storage-testbench)
12+
to configure and generate tests cases which use fault injection to ensure conformance.
13+
14+
## Running the Conformance Test Suite
15+
16+
#### Prerequisites
17+
1. Python 3.8
18+
2. Nox
19+
3. Docker
20+
21+
The Retry Strategy Conformance test suite is included in [`noxfile.py`](https://github.com/googleapis/python-storage/blob/main/noxfile.py) and run automatically as part of the Kokoro presubmits:
22+
1. Running the testbench server via docker
23+
2. Setup, validation, cleanup of individual test cases with the testbench
24+
3. Test logs included in Kokoro build
25+
26+
To run the test suite locally:
27+
```bash
28+
nox -s conftest_retry-3.8
29+
```

‎tests/conformance/retry_strategy_test_data.json

+113-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"retryStrategyTests": [
2+
"retryTests": [
33
{
44
"id": 1,
55
"description": "always_idempotent",
@@ -127,6 +127,118 @@
127127
],
128128
"preconditionProvided": false,
129129
"expectSuccess": false
130+
},
131+
{
132+
"id": 5,
133+
"description": "non_retryable_errors",
134+
"cases": [
135+
{
136+
"instructions": ["return-400"]
137+
},
138+
{
139+
"instructions": ["return-401"]
140+
}
141+
],
142+
"methods": [
143+
{"name": "storage.bucket_acl.delete", "resources": ["BUCKET"]},
144+
{"name": "storage.bucket_acl.get", "resources": ["BUCKET"]},
145+
{"name": "storage.bucket_acl.insert", "resources": ["BUCKET"]},
146+
{"name": "storage.bucket_acl.list", "resources": ["BUCKET"]},
147+
{"name": "storage.bucket_acl.patch", "resources": ["BUCKET"]},
148+
{"name": "storage.bucket_acl.update", "resources": ["BUCKET"]},
149+
{"name": "storage.buckets.delete", "resources": ["BUCKET"]},
150+
{"name": "storage.buckets.get", "resources": ["BUCKET"]},
151+
{"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]},
152+
{"name": "storage.buckets.insert", "resources": ["BUCKET"]},
153+
{"name": "storage.buckets.list", "resources": ["BUCKET"]},
154+
{"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]},
155+
{"name": "storage.buckets.patch", "resources": ["BUCKET"]},
156+
{"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]},
157+
{"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]},
158+
{"name": "storage.buckets.update", "resources": ["BUCKET"]},
159+
{"name": "storage.default_object_acl.delete", "resources": ["BUCKET"]},
160+
{"name": "storage.default_object_acl.get", "resources": ["BUCKET"]},
161+
{"name": "storage.default_object_acl.insert", "resources": ["BUCKET"]},
162+
{"name": "storage.default_object_acl.list", "resources": ["BUCKET"]},
163+
{"name": "storage.default_object_acl.patch", "resources": ["BUCKET"]},
164+
{"name": "storage.default_object_acl.update", "resources": ["BUCKET"]},
165+
{"name": "storage.hmacKey.create", "resources": []},
166+
{"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]},
167+
{"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]},
168+
{"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]},
169+
{"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]},
170+
{"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]},
171+
{"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]},
172+
{"name": "storage.notifications.insert", "resources": ["BUCKET", "NOTIFICATION"]},
173+
{"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]},
174+
{"name": "storage.object_acl.delete", "resources": ["BUCKET", "OBJECT"]},
175+
{"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]},
176+
{"name": "storage.object_acl.insert", "resources": ["BUCKET", "OBJECT"]},
177+
{"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]},
178+
{"name": "storage.object_acl.patch", "resources": ["BUCKET", "OBJECT"]},
179+
{"name": "storage.object_acl.update", "resources": ["BUCKET", "OBJECT"]},
180+
{"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]},
181+
{"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]},
182+
{"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]},
183+
{"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]},
184+
{"name": "storage.objects.insert", "resources": ["BUCKET"]},
185+
{"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]},
186+
{"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]},
187+
{"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]},
188+
{"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]},
189+
{"name": "storage.serviceaccount.get", "resources": []}
190+
],
191+
"preconditionProvided": false,
192+
"expectSuccess": false
193+
},
194+
{
195+
"id": 6,
196+
"description": "mix_retryable_non_retryable_errors",
197+
"cases": [
198+
{
199+
"instructions": ["return-503", "return-400"]
200+
},
201+
{
202+
"instructions": ["return-reset-connection", "return-401"]
203+
}
204+
],
205+
"methods": [
206+
{"name": "storage.bucket_acl.get", "resources": ["BUCKET"]},
207+
{"name": "storage.bucket_acl.list", "resources": ["BUCKET"]},
208+
{"name": "storage.buckets.delete", "resources": ["BUCKET"]},
209+
{"name": "storage.buckets.get", "resources": ["BUCKET"]},
210+
{"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]},
211+
{"name": "storage.buckets.insert", "resources": []},
212+
{"name": "storage.buckets.list", "resources": ["BUCKET"]},
213+
{"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]},
214+
{"name": "storage.buckets.patch", "resources": ["BUCKET"]},
215+
{"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]},
216+
{"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]},
217+
{"name": "storage.buckets.update", "resources": ["BUCKET"]},
218+
{"name": "storage.default_object_acl.get", "resources": ["BUCKET"]},
219+
{"name": "storage.default_object_acl.list", "resources": ["BUCKET"]},
220+
{"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]},
221+
{"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]},
222+
{"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]},
223+
{"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]},
224+
{"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]},
225+
{"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]},
226+
{"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]},
227+
{"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]},
228+
{"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]},
229+
{"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]},
230+
{"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]},
231+
{"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]},
232+
{"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]},
233+
{"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]},
234+
{"name": "storage.objects.insert", "resources": ["BUCKET"]},
235+
{"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]},
236+
{"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]},
237+
{"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]},
238+
{"name": "storage.serviceaccount.get", "resources": []}
239+
],
240+
"preconditionProvided": true,
241+
"expectSuccess": false
130242
}
131243
]
132244
}

‎tests/conformance/test_conformance.py

+7-8
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,16 @@
1414

1515
"""Conformance tests for retry. Verifies correct behavior around retryable errors, idempotency and preconditions."""
1616

17+
import functools
18+
import logging
1719
import os
18-
import requests
20+
import subprocess
1921
import tempfile
22+
import time
2023
import uuid
21-
import logging
22-
import functools
24+
2325
import pytest
24-
import subprocess
25-
import time
26+
import requests
2627

2728
from six.moves.urllib import parse as urlparse
2829

@@ -34,9 +35,7 @@
3435
from . import _read_local_json
3536

3637

37-
_CONFORMANCE_TESTS = _read_local_json("retry_strategy_test_data.json")[
38-
"retryStrategyTests"
39-
]
38+
_CONFORMANCE_TESTS = _read_local_json("retry_strategy_test_data.json")["retryTests"]
4039

4140
"""Environment variable or default host for Storage testbench emulator."""
4241
_HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000")

0 commit comments

Comments
 (0)