Skip to content

Commit 40e0f0b

Browse files
vaishakdineshvaishakdinesh
authored
Merge pull request #897 from cloudflare/vaishak/add-pagination
fix: pagination
2 parents e72362c + 1c2784c commit 40e0f0b

File tree

5 files changed

+331
-38
lines changed

5 files changed

+331
-38
lines changed

‎internal/app/cf-terraforming/cmd/custom_processing.go

Lines changed: 62 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"errors"
7+
"fmt"
78
"io"
89
"net/http"
910
"net/url"
@@ -317,50 +318,79 @@ func unMarshallJSONStructData(modifiedJSONString string) ([]interface{}, error)
317318
}
318319

319320
func getAPIResponse(result *http.Response, pathParams []string, endpoints ...string) ([]interface{}, error) {
320-
var jsonStructData, results []interface{}
321-
for i, endpoint := range endpoints {
322-
err := api.Get(context.Background(), endpoint, nil, &result)
323-
if err != nil {
324-
var apierr *cloudflare.Error
325-
if errors.As(err, &apierr) {
326-
if apierr.StatusCode == http.StatusNotFound {
321+
var allResults []interface{}
322+
323+
for i, baseEndpoint := range endpoints {
324+
page := 1
325+
totalPages := 1
326+
param := ""
327+
if len(pathParams) > 0 {
328+
param = pathParams[i]
329+
}
330+
331+
for {
332+
var endpoint string
333+
// no page param for first request
334+
if page == 1 {
335+
endpoint = baseEndpoint
336+
} else {
337+
sep := "?"
338+
if strings.Contains(baseEndpoint, "?") {
339+
sep = "&"
340+
}
341+
endpoint = fmt.Sprintf("%s%spage=%d", baseEndpoint, sep, page)
342+
}
343+
344+
err := api.Get(context.Background(), endpoint, nil, &result)
345+
if err != nil {
346+
var apierr *cloudflare.Error
347+
if errors.As(err, &apierr) && apierr.StatusCode == http.StatusNotFound {
327348
log.WithFields(logrus.Fields{
328349
"resource": resourceType,
329350
"endpoint": endpoint,
330351
}).Debug("no resources found")
331352
return nil, err
332353
}
354+
log.Fatalf("failed to fetch API endpoint: %s", err)
333355
}
334-
log.Fatalf("failed to fetch API endpoint: %s", err)
335-
}
336356

337-
body, err := io.ReadAll(result.Body)
338-
if err != nil {
339-
log.Fatalln(err)
340-
}
341-
value := gjson.Get(string(body), "result")
342-
if value.Type == gjson.Null {
343-
log.WithFields(logrus.Fields{
344-
"resource": resourceType,
345-
"endpoint": endpoint,
346-
}).Debug("no result found")
347-
return nil, errors.New("no result found")
348-
}
357+
body, err := io.ReadAll(result.Body)
358+
if err != nil {
359+
log.Fatalln(err)
360+
}
349361

350-
modifiedJSON := modifyResponsePayload(resourceType, value)
351-
jsonStructData, err = unMarshallJSONStructData(modifiedJSON)
352-
if err != nil {
353-
log.Fatalf("failed to unmarshal result: %s", err)
354-
}
362+
resultVal := gjson.Get(string(body), "result")
363+
if resultVal.Type == gjson.Null {
364+
log.WithFields(logrus.Fields{
365+
"resource": resourceType,
366+
"endpoint": endpoint,
367+
}).Debug("no result found")
368+
return nil, errors.New("no result found")
369+
}
355370

356-
param := ""
357-
if len(pathParams) > 0 {
358-
param = pathParams[i]
371+
modifiedJSON := modifyResponsePayload(resourceType, resultVal)
372+
jsonStructData, err := unMarshallJSONStructData(modifiedJSON)
373+
if err != nil {
374+
log.Fatalf("failed to unmarshal result: %s", err)
375+
}
376+
377+
processCustomCasesV5(&jsonStructData, resourceType, param)
378+
allResults = append(allResults, jsonStructData...)
379+
380+
if page == 1 {
381+
totalPagesVal := gjson.Get(string(body), "result_info.total_pages")
382+
if totalPagesVal.Exists() {
383+
totalPages = int(totalPagesVal.Int())
384+
}
385+
}
386+
387+
if page >= totalPages {
388+
break
389+
}
390+
page++
359391
}
360-
processCustomCasesV5(&jsonStructData, resourceType, param)
361-
results = append(results, jsonStructData...)
362392
}
363-
return results, nil
393+
return allResults, nil
364394
}
365395

366396
func isSupportedPathParam(resources []string, rType string) bool {

‎testdata/cloudflare/v5/cloudflare_account_member.yaml

Lines changed: 250 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ version: 1
33
interactions:
44
- request:
55
body: ""
6-
form: {}
6+
form: { }
77
headers:
88
Accept:
99
- application/json
@@ -95,7 +95,131 @@ interactions:
9595
"result_info":{
9696
"page":1,
9797
"per_page":15,
98-
"total_pages":2,
98+
"total_pages":3,
99+
"count":15,
100+
"total_count":18
101+
},
102+
"success":true,
103+
"errors":[
104+
105+
],
106+
"messages":[
107+
108+
]
109+
}
110+
headers:
111+
Allow:
112+
- GET, POST
113+
Cache-Control:
114+
- private,no-cache,no-store
115+
Cf-Auditlog-Id:
116+
- 01951b40-03ef-736b-b60f-86e9bf38f85c
117+
Connection:
118+
- keep-alive
119+
Content-Type:
120+
- application/json
121+
Pragma:
122+
- no-cache
123+
Vary:
124+
- Accept-Encoding
125+
status: 200 OK
126+
code: 200
127+
duration: ""
128+
- request:
129+
body: ""
130+
form: {}
131+
headers:
132+
Accept:
133+
- application/json
134+
X-Stainless-Arch:
135+
- arm64
136+
X-Stainless-Lang:
137+
- go
138+
X-Stainless-Os:
139+
- MacOS
140+
X-Stainless-Package-Version:
141+
- 4.0.0
142+
X-Stainless-Retry-Count:
143+
- "0"
144+
X-Stainless-Runtime:
145+
- go
146+
X-Stainless-Runtime-Version:
147+
- go1.23.5
148+
url: https://api.cloudflare.com/client/v4/accounts/f037e56e89293a057740de681ac9abbe/members?page=2
149+
method: GET
150+
response:
151+
body: |
152+
{
153+
"result":[
154+
{
155+
"id":"9ad9fa82d0ca4f4da16d8869b062892b",
156+
"user":{
157+
"id":"fffe8afdb4bd5b94c5b093a23a741135",
158+
"first_name":"John",
159+
"last_name":"doe",
160+
"email":"[email protected]",
161+
"two_factor_authentication_enabled":true
162+
},
163+
"status":"accepted",
164+
"api_access_enabled":null,
165+
"policies":[
166+
{
167+
"id":"aa0fe1963f8f49c8a4f2d73314146e23",
168+
"access":"allow",
169+
"permission_groups":[
170+
{
171+
"id":"8e23b19e4e0d44c29d239c5688ba8cbb",
172+
"name":"Super Administrator - All Privileges",
173+
"meta":{
174+
"description":"Can edit any Cloudflare setting, make purchases, update billing, and manage memberships. Super Administrators can revoke the access of other Super Administrators.",
175+
"editable":"false",
176+
"label":"all_privileges",
177+
"scopes":"com.cloudflare.api.account"
178+
}
179+
}
180+
],
181+
"resource_groups":[
182+
{
183+
"id":"68dfcdfb261c4d48aa75ec0e5413f2c3",
184+
"name":"com.cloudflare.api.account.f037e56e89293a057740de681ac9abbe",
185+
"meta":{
186+
"editable":"false"
187+
},
188+
"scope":{
189+
"key":"com.cloudflare.api.account.f037e56e89293a057740de681ac9abbe",
190+
"objects":[
191+
{
192+
"key":"*"
193+
}
194+
]
195+
}
196+
}
197+
]
198+
}
199+
],
200+
"roles":[
201+
{
202+
"id":"33666b9c79b9a5273fc7344ff42f953d",
203+
"name":"Super Administrator - All Privileges",
204+
"description":"Can edit any Cloudflare setting, make purchases, update billing, and manage memberships. Super Administrators can revoke the access of other Super Administrators.",
205+
"permissions":{
206+
"access":{
207+
"edit":true,
208+
"read":true
209+
},
210+
"analytics":{
211+
"edit":false,
212+
"read":true
213+
}
214+
}
215+
}
216+
]
217+
}
218+
],
219+
"result_info":{
220+
"page":2,
221+
"per_page":15,
222+
"total_pages":3,
99223
"count":15,
100224
"total_count":18
101225
},
@@ -125,3 +249,127 @@ interactions:
125249
status: 200 OK
126250
code: 200
127251
duration: ""
252+
- request:
253+
body: ""
254+
form: { }
255+
headers:
256+
Accept:
257+
- application/json
258+
X-Stainless-Arch:
259+
- arm64
260+
X-Stainless-Lang:
261+
- go
262+
X-Stainless-Os:
263+
- MacOS
264+
X-Stainless-Package-Version:
265+
- 4.0.0
266+
X-Stainless-Retry-Count:
267+
- "0"
268+
X-Stainless-Runtime:
269+
- go
270+
X-Stainless-Runtime-Version:
271+
- go1.23.5
272+
url: https://api.cloudflare.com/client/v4/accounts/f037e56e89293a057740de681ac9abbe/members?page=3
273+
method: GET
274+
response:
275+
body: |
276+
{
277+
"result":[
278+
{
279+
"id":"9ad9fa82d0ca4f4da16d8869b062892b",
280+
"user":{
281+
"id":"fffe8afdb4bd5b94c5b093a23a741135",
282+
"first_name":"John",
283+
"last_name":"doe",
284+
"email":"[email protected]",
285+
"two_factor_authentication_enabled":true
286+
},
287+
"status":"accepted",
288+
"api_access_enabled":null,
289+
"policies":[
290+
{
291+
"id":"aa0fe1963f8f49c8a4f2d73314146e23",
292+
"access":"allow",
293+
"permission_groups":[
294+
{
295+
"id":"8e23b19e4e0d44c29d239c5688ba8cbb",
296+
"name":"Super Administrator - All Privileges",
297+
"meta":{
298+
"description":"Can edit any Cloudflare setting, make purchases, update billing, and manage memberships. Super Administrators can revoke the access of other Super Administrators.",
299+
"editable":"false",
300+
"label":"all_privileges",
301+
"scopes":"com.cloudflare.api.account"
302+
}
303+
}
304+
],
305+
"resource_groups":[
306+
{
307+
"id":"68dfcdfb261c4d48aa75ec0e5413f2c3",
308+
"name":"com.cloudflare.api.account.f037e56e89293a057740de681ac9abbe",
309+
"meta":{
310+
"editable":"false"
311+
},
312+
"scope":{
313+
"key":"com.cloudflare.api.account.f037e56e89293a057740de681ac9abbe",
314+
"objects":[
315+
{
316+
"key":"*"
317+
}
318+
]
319+
}
320+
}
321+
]
322+
}
323+
],
324+
"roles":[
325+
{
326+
"id":"33666b9c79b9a5273fc7344ff42f953d",
327+
"name":"Super Administrator - All Privileges",
328+
"description":"Can edit any Cloudflare setting, make purchases, update billing, and manage memberships. Super Administrators can revoke the access of other Super Administrators.",
329+
"permissions":{
330+
"access":{
331+
"edit":true,
332+
"read":true
333+
},
334+
"analytics":{
335+
"edit":false,
336+
"read":true
337+
}
338+
}
339+
}
340+
]
341+
}
342+
],
343+
"result_info":{
344+
"page":3,
345+
"per_page":15,
346+
"total_pages":3,
347+
"count":15,
348+
"total_count":18
349+
},
350+
"success":true,
351+
"errors":[
352+
353+
],
354+
"messages":[
355+
356+
]
357+
}
358+
headers:
359+
Allow:
360+
- GET, POST
361+
Cache-Control:
362+
- private,no-cache,no-store
363+
Cf-Auditlog-Id:
364+
- 01951b40-03ef-736b-b60f-86e9bf38f85c
365+
Connection:
366+
- keep-alive
367+
Content-Type:
368+
- application/json
369+
Pragma:
370+
- no-cache
371+
Vary:
372+
- Accept-Encoding
373+
status: 200 OK
374+
code: 200
375+
duration: ""

‎testdata/cloudflare/v5/cloudflare_dns_record.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ interactions:
104104
"per_page": 100,
105105
"count": 100,
106106
"total_count": 193,
107-
"total_pages": 2
107+
"total_pages": 1
108108
}
109109
}
110110
headers:

0 commit comments

Comments
 (0)