Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🐛 Dependency-Pinning: only score detected ecosystems #3436

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
2b2dd82
feat: Define if dependency is pinned or unpinned
gabibguti Feb 14, 2023
9638a92
refactor: Convert diff var types to pointer
gabibguti Feb 16, 2023
e497a20
fix: Pinned Dependency field type
gabibguti Feb 16, 2023
3f3dd80
feat: Count pinned and unpinned deps
gabibguti Aug 28, 2023
309158a
feat: Flag not applicable ecossystems
gabibguti Aug 28, 2023
14f69b1
feat: Score only applicable ecossystems
gabibguti Aug 28, 2023
6ff571b
feat: If no dependencies then create inconclusive score
gabibguti Aug 28, 2023
7b15634
test: GitHub Actions score and logs
gabibguti Aug 29, 2023
104f969
test: Pinned dependencies score
gabibguti Aug 29, 2023
5f67732
test: Ecossystems score and logs
gabibguti Aug 29, 2023
5691e0d
test: Remove deleted maxScore function test
gabibguti Aug 29, 2023
84757ea
test: Adding GitHub Actions dependencies to result
gabibguti Aug 29, 2023
5509c47
test: Update GitHub Actions result
gabibguti Aug 29, 2023
9a787c4
test: Update pip installs result
gabibguti Aug 29, 2023
423cac8
fix: Handle if nuget dependency is pinned or unpinned
gabibguti Apr 18, 2023
9056875
tests: Fix check warnings for unpinned dependencies
gabibguti Aug 29, 2023
c1a81fe
fix: Linter errors
gabibguti Aug 29, 2023
c8ee14a
fix: GitHub Actions pinned log
gabibguti Aug 29, 2023
7f7b89d
test: Fix "ossf-tests/scorecard-check-pinned-dependencies-e2e"
gabibguti Aug 29, 2023
a1b1781
Revert rename `asPointer` to `asStringPointer`
gabibguti Aug 31, 2023
5adf329
fix: Handle deps with parsing error and undefined pinning
gabibguti Aug 31, 2023
1ad766b
test: Delete unecessary test
gabibguti Aug 31, 2023
7a2a3a4
test: Add missing dep Location cases
gabibguti Aug 31, 2023
4412f0e
fix: Simplify Dockerfile pinned as name logic
gabibguti Aug 31, 2023
512659c
fix: If ecossystem is not found show debug log
gabibguti Aug 31, 2023
9731d8a
test: Fix e2e tests and more unit tests
gabibguti Aug 31, 2023
08bd85e
feat: Iterate all dependency types for final score
gabibguti Aug 31, 2023
e52be0b
feat: Proportional score
gabibguti Sep 4, 2023
29be827
fix: GHA weights in proportional score
gabibguti Sep 4, 2023
5dc6464
test: Fix scores and logs checking
gabibguti Sep 4, 2023
e8667cc
test: Fix e2e test
gabibguti Sep 4, 2023
407773c
refactor: Rename to ProportionalScoreWeighted
gabibguti Sep 6, 2023
a68194b
refactor: Var declarations to create proportional score
gabibguti Sep 6, 2023
aa23b15
fix: Remove unnecessary pointer
gabibguti Sep 6, 2023
e478307
fix: Dependencies priority declaration
gabibguti Sep 6, 2023
99f4dc6
fix: Ecosystem spelling
gabibguti Sep 13, 2023
7000eb5
fix: Handle 0 weight and 0 total when creating proportional weighted …
gabibguti Sep 13, 2023
6390493
fix: Revert -d flag identification change
gabibguti Sep 13, 2023
3187362
fix: npm ci command is npm download and is pinned
gabibguti Sep 13, 2023
2484ddc
fix: Linter errors
gabibguti Sep 13, 2023
24f3a8f
fix: Unexport error variable to other packages
gabibguti Sep 15, 2023
f839462
refactor: Simplify no score groups condition
gabibguti Sep 15, 2023
ca725e0
feat: Log proportion of dependencies pinned
gabibguti Sep 19, 2023
f6ccf91
test: Fix unit tests to include info logs
gabibguti Sep 19, 2023
9baf547
test: Fix e2e tests to include info logs
gabibguti Sep 19, 2023
b986c65
fix: Linter error
gabibguti Sep 19, 2023
2e3e9a5
Merge branch 'main' into fix/count-existing-ecossystems-for-pinned-deps
spencerschrock Sep 25, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions checker/check_result.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package checker

import (
"errors"
"fmt"
"math"

Expand Down Expand Up @@ -50,6 +51,10 @@ const (
DetailDebug
)

// errSuccessTotal indicates a runtime error because number of success cases should
// be smaller than the total cases to create a proportional score.
var errSuccessTotal = errors.New("unexpected number of success is higher than total")

// CheckResult captures result from a check run.
//
//nolint:govet
Expand Down Expand Up @@ -88,6 +93,14 @@ type LogMessage struct {
Remediation *rule.Remediation // Remediation information, if any.
}

// ProportionalScoreWeighted is a structure that contains
// the fields to calculate weighted proportional scores.
type ProportionalScoreWeighted struct {
Success int
Total int
Weight int
}

// CreateProportionalScore creates a proportional score.
func CreateProportionalScore(success, total int) int {
if total == 0 {
Expand All @@ -97,6 +110,40 @@ func CreateProportionalScore(success, total int) int {
return int(math.Min(float64(MaxResultScore*success/total), float64(MaxResultScore)))
}

// CreateProportionalScoreWeighted creates the proportional score
// between multiple successes over the total, but some proportions
// are worth more.
func CreateProportionalScoreWeighted(scores ...ProportionalScoreWeighted) (int, error) {
var ws, wt int
allWeightsZero := true
noScoreGroups := true
for _, score := range scores {
if score.Success > score.Total {
return InconclusiveResultScore, fmt.Errorf("%w: %d, %d", errSuccessTotal, score.Success, score.Total)
}
if score.Total == 0 {
continue // Group with 0 total, does not count for score
}
noScoreGroups = false
if score.Weight != 0 {
allWeightsZero = false
}
// Group with zero weight, adds nothing to the score

ws += score.Success * score.Weight
wt += score.Total * score.Weight
}
if noScoreGroups {
return InconclusiveResultScore, nil
}
// If has score groups but no groups matter to the score, result in max score
if allWeightsZero {
return MaxResultScore, nil
}

return int(math.Min(float64(MaxResultScore*ws/wt), float64(MaxResultScore))), nil
}

// AggregateScores adds up all scores
// and normalizes the result.
// Each score contributes equally.
Expand Down
267 changes: 267 additions & 0 deletions checker/check_result_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,273 @@ func TestCreateProportionalScore(t *testing.T) {
}
}

func TestCreateProportionalScoreWeighted(t *testing.T) {
t.Parallel()
type want struct {
score int
err bool
}
tests := []struct {
name string
scores []ProportionalScoreWeighted
want want
}{
{
name: "max result with 1 group and normal weight",
scores: []ProportionalScoreWeighted{
{
Success: 1,
Total: 1,
Weight: 10,
},
},
want: want{
score: 10,
},
},
{
name: "min result with 1 group and normal weight",
scores: []ProportionalScoreWeighted{
{
Success: 0,
Total: 1,
Weight: 10,
},
},
want: want{
score: 0,
},
},
{
name: "partial result with 1 group and normal weight",
scores: []ProportionalScoreWeighted{
{
Success: 2,
Total: 10,
Weight: 10,
},
},
want: want{
score: 2,
},
},
{
name: "partial result with 2 groups and normal weights",
scores: []ProportionalScoreWeighted{
{
Success: 2,
Total: 10,
Weight: 10,
},
{
Success: 8,
Total: 10,
Weight: 10,
},
},
want: want{
score: 5,
},
},
{
name: "partial result with 2 groups and odd weights",
scores: []ProportionalScoreWeighted{
{
Success: 2,
Total: 10,
Weight: 8,
},
{
Success: 8,
Total: 10,
Weight: 2,
},
},
want: want{
score: 3,
},
},
{
name: "all groups with 0 weight, no groups matter for the score, results in max score",
scores: []ProportionalScoreWeighted{
{
Success: 1,
Total: 1,
Weight: 0,
},
{
Success: 1,
Total: 1,
Weight: 0,
},
},
want: want{
score: 10,
},
},
{
name: "not all groups with 0 weight, only groups with weight matter to the score",
scores: []ProportionalScoreWeighted{
{
Success: 1,
Total: 1,
Weight: 0,
},
{
Success: 2,
Total: 10,
Weight: 8,
},
{
Success: 8,
Total: 10,
Weight: 2,
},
},
want: want{
score: 3,
},
},
{
name: "no total, results in inconclusive score",
scores: []ProportionalScoreWeighted{
{
Success: 0,
Total: 0,
Weight: 10,
},
},
want: want{
score: -1,
},
},
{
name: "some groups with 0 total, only groups with total matter to the score",
scores: []ProportionalScoreWeighted{
{
Success: 0,
Total: 0,
Weight: 10,
},
{
Success: 2,
Total: 10,
Weight: 10,
},
},
want: want{
score: 2,
},
},
{
name: "any group with number of successes higher than total, results in inconclusive score and error",
scores: []ProportionalScoreWeighted{
{
Success: 1,
Total: 0,
Weight: 10,
},
},
want: want{
score: -1,
err: true,
},
},
{
name: "only groups with weight and total matter to the score",
scores: []ProportionalScoreWeighted{
{
Success: 1,
Total: 1,
Weight: 0,
},
{
Success: 0,
Total: 0,
Weight: 10,
},
{
Success: 2,
Total: 10,
Weight: 8,
},
{
Success: 8,
Total: 10,
Weight: 2,
},
},
want: want{
score: 3,
},
},
{
name: "only groups with weight and total matter to the score but no groups have success, results in min score",
scores: []ProportionalScoreWeighted{
{
Success: 1,
Total: 1,
Weight: 0,
},
{
Success: 0,
Total: 0,
Weight: 10,
},
{
Success: 0,
Total: 10,
Weight: 8,
},
{
Success: 0,
Total: 10,
Weight: 2,
},
},
want: want{
score: 0,
},
},
{
name: "group with 0 weight counts as max score and group with 0 total does not count",
scores: []ProportionalScoreWeighted{
{
Success: 2,
Total: 8,
Weight: 0,
},
{
Success: 0,
Total: 0,
Weight: 10,
},
},
want: want{
score: 10,
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := CreateProportionalScoreWeighted(tt.scores...)
if err != nil && !tt.want.err {
t.Errorf("CreateProportionalScoreWeighted unexpected error '%v'", err)
t.Fail()
}
if err == nil && tt.want.err {
t.Errorf("CreateProportionalScoreWeighted expected error and got none")
t.Fail()
}
if got != tt.want.score {
t.Errorf("CreateProportionalScoreWeighted() = %v, want %v", got, tt.want.score)
}
})
}
}

func TestNormalizeReason(t *testing.T) {
t.Parallel()
type args struct {
Expand Down
1 change: 1 addition & 0 deletions checker/raw_result.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ type Dependency struct {
PinnedAt *string
Location *File
Msg *string // Only for debug messages.
Pinned *bool
spencerschrock marked this conversation as resolved.
Show resolved Hide resolved
Type DependencyUseType
}

Expand Down
Loading
Loading