Skip to content

Commit

Permalink
fix: confluence anonymous access is failing after few requests (#146)
Browse files Browse the repository at this point in the history 
Fixes #144

The error is not always reproduced, and when it is, it is reproduced for
a different page every time.

Since the error is `500`, means that it is a server (Confluence) error,
I can't fix it and I must work around it.

I added a retry mechanism and I enabled it only for our request, when it
returns `500`, it will retry it 3 times.

It looks like this:
```
11:38:19 WRN confluence credentials were not provided. The scan will be made anonymously only for the public pages
11:38:20 INF  Total of all 4 Spaces detected
11:38:20 INF  Space - secrets have 3 pages
11:38:20 INF  Space - Secrets have 1 pages
11:38:20 INF  Space - Checkmarx Resources have 3 pages
11:38:29 INF  Space - Checkmarx Knowledge Center have 1231 pages
11:38:37 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1375995197?expand=body.storage,version,history.previousVersion
11:38:37 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1312260170?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1188135832?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2093908162?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1829044450?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1534427435?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1383432225?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1191609716?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1636598107?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2031092718?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1998946990?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1995211335?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2100003058?expand=body.storage,version,history.previousVersion
11:38:38 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1192362348?expand=body.storage,version,history.previousVersion
11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1169195803?expand=body.storage,version,history.previousVersion
11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1131742701?expand=body.storage,version,history.previousVersion
11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1947631705?expand=body.storage,version,history.previousVersion
11:38:40 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1158283441?expand=body.storage,version,history.previousVersion
11:38:41 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2490630327?expand=body.storage,version,history.previousVersion
11:38:42 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/2600012551?expand=body.storage,version,history.previousVersion
11:38:39 WRN retrying http request https://checkmarx.atlassian.net/wiki/rest/api/content/1186267153?expand=body.storage,version,history.previousVersion
exit status 1
Summary:
totalitemsscanned: 1238
totalsecretsfound: 6
```
We can see the `500` error code happens once a time and then all the
requests will return with `500`.
  • Loading branch information
Baruch Odem (Rothkoff) authored Jul 26, 2023
1 parent a76401d commit e6ab56c
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 7 deletions.
21 changes: 18 additions & 3 deletions lib/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"fmt"
"io"
"net/http"

"github.com/rs/zerolog/log"
)

type ICredentials interface {
Expand All @@ -20,14 +22,19 @@ type IAuthorizationHeader interface {
GetAuthorizationHeader() string
}

func HttpRequest(method string, url string, autherization IAuthorizationHeader) ([]byte, *http.Response, error) {
type RetrySettings struct {
MaxRetries int
ErrorCodes []int
}

func HttpRequest(method string, url string, authorization IAuthorizationHeader, retry RetrySettings) ([]byte, *http.Response, error) {
request, err := http.NewRequest(method, url, nil)
if err != nil {
return nil, nil, fmt.Errorf("unexpected error creating an http request %w", err)
}

if autherization.GetAuthorizationHeader() != "" {
request.Header.Set("Authorization", autherization.GetAuthorizationHeader())
if authorization.GetAuthorizationHeader() != "" {
request.Header.Set("Authorization", authorization.GetAuthorizationHeader())
}

client := &http.Client{}
Expand All @@ -39,6 +46,14 @@ func HttpRequest(method string, url string, autherization IAuthorizationHeader)
defer response.Body.Close()

if response.StatusCode < 200 || response.StatusCode >= 300 {
if retry.MaxRetries > 0 {
for _, code := range retry.ErrorCodes {
if response.StatusCode == code {
log.Warn().Msgf("retrying http request %v", url)
return HttpRequest(method, url, authorization, RetrySettings{MaxRetries: retry.MaxRetries - 1, ErrorCodes: retry.ErrorCodes})
}
}
}
return nil, response, fmt.Errorf("error calling http url \"%v\". status code: %v", url, response)
}

Expand Down
6 changes: 3 additions & 3 deletions plugins/confluence.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) {

func (p *ConfluencePlugin) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) {
url := fmt.Sprintf("%s/rest/api/space?start=%d", p.URL, start)
body, _, err := lib.HttpRequest(http.MethodGet, url, p)
body, _, err := lib.HttpRequest(http.MethodGet, url, p, lib.RetrySettings{})
if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
}
Expand Down Expand Up @@ -206,7 +206,7 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag

func (p *ConfluencePlugin) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) {
url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", p.URL, space.Key, start)
body, _, err := lib.HttpRequest(http.MethodGet, url, p)
body, _, err := lib.HttpRequest(http.MethodGet, url, p, lib.RetrySettings{})

if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
Expand Down Expand Up @@ -253,7 +253,7 @@ func (p *ConfluencePlugin) getItem(page ConfluencePage, space ConfluenceSpaceRes
url = fmt.Sprintf("%s/rest/api/content/%s?status=historical&version=%d&expand=body.storage,version,history.previousVersion", p.URL, page.ID, version)
}

request, _, err := lib.HttpRequest(http.MethodGet, url, p)
request, _, err := lib.HttpRequest(http.MethodGet, url, p, lib.RetrySettings{MaxRetries: 3, ErrorCodes: []int{500}})
if err != nil {
return nil, 0, fmt.Errorf("unexpected error creating an http request %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion plugins/paligo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ func (p *PaligoClient) request(endpoint string, lim *rate.Limiter) ([]byte, erro
}

url := fmt.Sprintf("https://%s.paligoapp.com/api/v2/%s", p.Instance, endpoint)
body, response, err := lib.HttpRequest("GET", url, p.auth)
body, response, err := lib.HttpRequest("GET", url, p.auth, lib.RetrySettings{})
if err != nil {
if err := reserveRateLimit(response, lim, err); err != nil {
return nil, err
Expand Down

0 comments on commit e6ab56c

Please sign in to comment.