Automatically add per_page=100 to paginated REST requests

mislav · mislav · commit c945fb4336ff · 2020-06-23T18:42:57.000+02:00
Most endpoints respect this parameter by default. Those that don't will
just ignore it. The `per_page=100` parameter is not added if there is
already a `per_page` parameter specified in the request.
diff --git a/pkg/cmd/api/api.go b/pkg/cmd/api/api.go
@@ -143,6 +143,7 @@ func apiRun(opts *ApiOptions) error {
 		return err
 	}
 
+	isGraphQL := opts.RequestPath == "graphql"
 	requestPath, err := fillPlaceholders(opts.RequestPath, opts)
 	if err != nil {
 		return fmt.Errorf("unable to expand placeholder in path: %w", err)
@@ -155,6 +156,10 @@ func apiRun(opts *ApiOptions) error {
 		method = "POST"
 	}
 
+	if opts.Paginate && !isGraphQL {
+		requestPath = addPerPage(requestPath, 100, params)
+	}
+
 	if opts.RequestInputFile != "" {
 		file, size, err := openUserFile(opts.RequestInputFile, opts.IO.In)
 		if err != nil {
@@ -189,7 +194,7 @@ func apiRun(opts *ApiOptions) error {
 			break
 		}
 
-		if opts.RequestPath == "graphql" {
+		if isGraphQL {
 			hasNextPage = endCursor != ""
 			if hasNextPage {
 				params["endCursor"] = endCursor
diff --git a/pkg/cmd/api/api_test.go b/pkg/cmd/api/api_test.go
@@ -343,7 +343,7 @@ func Test_apiRun_paginationREST(t *testing.T) {
 	assert.Equal(t, `{"page":1}{"page":2}{"page":3}`, stdout.String(), "stdout")
 	assert.Equal(t, "", stderr.String(), "stderr")
 
-	assert.Equal(t, "https://api.github.com/issues", responses[0].Request.URL.String())
+	assert.Equal(t, "https://api.github.com/issues?per_page=100", responses[0].Request.URL.String())
 	assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=2", responses[1].Request.URL.String())
 	assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=3", responses[2].Request.URL.String())
 }
diff --git a/pkg/cmd/api/pagination.go b/pkg/cmd/api/pagination.go
@@ -2,9 +2,12 @@ package api
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"regexp"
+	"strings"
 )
 
 var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`)
@@ -85,3 +88,21 @@ loop:
 	}
 	return ""
 }
+
+func addPerPage(p string, perPage int, params map[string]interface{}) string {
+	if _, hasPerPage := params["per_page"]; hasPerPage {
+		return p
+	}
+
+	idx := strings.IndexRune(p, '?')
+	sep := "?"
+
+	if idx >= 0 {
+		if qp, err := url.ParseQuery(p[idx+1:]); err == nil && qp.Get("per_page") != "" {
+			return p
+		}
+		sep = "&"
+	}
+
+	return fmt.Sprintf("%s%sper_page=%d", p, sep, perPage)
+}
diff --git a/pkg/cmd/api/pagination_test.go b/pkg/cmd/api/pagination_test.go
@@ -116,3 +116,54 @@ func Test_findEndCursor(t *testing.T) {
 		})
 	}
 }
+
+func Test_addPerPage(t *testing.T) {
+	type args struct {
+		p       string
+		perPage int
+		params  map[string]interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "adds per_page",
+			args: args{
+				p:       "items",
+				perPage: 13,
+				params:  nil,
+			},
+			want: "items?per_page=13",
+		},
+		{
+			name: "avoids adding per_page if already in params",
+			args: args{
+				p:       "items",
+				perPage: 13,
+				params: map[string]interface{}{
+					"state":    "open",
+					"per_page": 99,
+				},
+			},
+			want: "items",
+		},
+		{
+			name: "avoids adding per_page if already in query",
+			args: args{
+				p:       "items?per_page=6&state=open",
+				perPage: 13,
+				params:  nil,
+			},
+			want: "items?per_page=6&state=open",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := addPerPage(tt.args.p, tt.args.perPage, tt.args.params); got != tt.want {
+				t.Errorf("addPerPage() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -343,7 +343,7 @@ func Test_apiRun_paginationREST(t *testing.T) {`
`343`	`343`	assert.Equal(t, `{"page":1}{"page":2}{"page":3}`, stdout.String(), "stdout")
`344`	`344`	`assert.Equal(t, "", stderr.String(), "stderr")`
`345`	`345`
`346`		`- assert.Equal(t, "https://api.github.com/issues", responses[0].Request.URL.String())`
	`346`	`+ assert.Equal(t, "https://api.github.com/issues?per_page=100", responses[0].Request.URL.String())`
`347`	`347`	`assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=2", responses[1].Request.URL.String())`
`348`	`348`	`assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=3", responses[2].Request.URL.String())`
`349`	`349`	`}`