Skip to content

Commit

Permalink
Add MAX_CACHE_SIZE config (#344)
Browse files Browse the repository at this point in the history
* Add MAX_CACHE_SIZE

* Bump actions version

* Optimize code

* Use Ticker to control GC loop
  • Loading branch information
n0vad3v authored Jun 29, 2024
1 parent f207e3c commit 89ea0af
Show file tree
Hide file tree
Showing 16 changed files with 208 additions and 69 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/CI.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Install Go
uses: actions/setup-go@v3
uses: actions/setup-go@v5
with:
go-version: '1.22'

Expand All @@ -35,7 +35,7 @@ jobs:
pull-requests: write
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/codecov.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Install Go
uses: actions/setup-go@v3
uses: actions/setup-go@v5
with:
go-version: '1.22'

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Install Go
uses: actions/setup-go@v3
uses: actions/setup-go@v5
with:
go-version: '1.22'

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/integration-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Build image
uses: docker/build-push-action@v3
Expand Down Expand Up @@ -43,7 +43,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Build image
uses: docker/build-push-action@v3
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/release_binary.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
uses: actions/checkout@v4

- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3

- name: Make WebP Server Go (amd64)
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release_docker_image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: true

Expand Down
3 changes: 1 addition & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ tools-dir:

install-staticcheck: tools-dir
GOBIN=`pwd`/tools/bin go install honnef.co/go/tools/cmd/staticcheck@latest
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.52.2
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.59.1

static-check: install-staticcheck
#S1000,SA1015,SA4006,SA4011,S1023,S1034,ST1003,ST1005,ST1016,ST1020,ST1021
Expand All @@ -39,6 +39,5 @@ test:
clean:
rm -rf prefetch remote-raw exhaust tools coverage.txt metadata exhaust_test


docker:
DOCKER_BUILDKIT=1 docker build -t webpsh/webps .
64 changes: 40 additions & 24 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ const (
"READ_BUFFER_SIZE": 4096,
"CONCURRENCY": 262144,
"DISABLE_KEEPALIVE": false,
"CACHE_TTL": 259200
"CACHE_TTL": 259200,
"MAX_CACHE_SIZE": 0
}`
)

Expand All @@ -50,11 +51,9 @@ var (
ProxyMode bool
Prefetch bool
Config = NewWebPConfig()
Version = "0.11.3"
Version = "0.11.4"
WriteLock = cache.New(5*time.Minute, 10*time.Minute)
ConvertLock = cache.New(5*time.Minute, 10*time.Minute)
RemoteRaw = "./remote-raw"
Metadata = "./metadata"
LocalHostAlias = "local"
RemoteCache *cache.Cache
)
Expand All @@ -66,14 +65,16 @@ type MetaFile struct {
}

type WebpConfig struct {
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ConvertTypes []string `json:"CONVERT_TYPES"`
ImageMap map[string]string `json:"IMG_MAP"`
ExhaustPath string `json:"EXHAUST_PATH"`
Host string `json:"HOST"`
Port string `json:"PORT"`
ImgPath string `json:"IMG_PATH"`
Quality int `json:"QUALITY,string"`
AllowedTypes []string `json:"ALLOWED_TYPES"`
ConvertTypes []string `json:"CONVERT_TYPES"`
ImageMap map[string]string `json:"IMG_MAP"`
ExhaustPath string `json:"EXHAUST_PATH"`
MetadataPath string `json:"METADATA_PATH"`
RemoteRawPath string `json:"REMOTE_RAW_PATH"`

EnableWebP bool `json:"ENABLE_WEBP"`
EnableAVIF bool `json:"ENABLE_AVIF"`
Expand All @@ -86,19 +87,23 @@ type WebpConfig struct {
ReadBufferSize int `json:"READ_BUFFER_SIZE"`
Concurrency int `json:"CONCURRENCY"`
DisableKeepalive bool `json:"DISABLE_KEEPALIVE"`
CacheTTL int `json:"CACHE_TTL"`
CacheTTL int `json:"CACHE_TTL"` // In minutes

MaxCacheSize int `json:"MAX_CACHE_SIZE"` // In MB, for max cached exhausted/metadata files(plus remote-raw if applicable), 0 means no limit
}

func NewWebPConfig() *WebpConfig {
return &WebpConfig{
Host: "0.0.0.0",
Port: "3333",
ImgPath: "./pics",
Quality: 80,
AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
ConvertTypes: []string{"webp"},
ImageMap: map[string]string{},
ExhaustPath: "./exhaust",
Host: "0.0.0.0",
Port: "3333",
ImgPath: "./pics",
Quality: 80,
AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
ConvertTypes: []string{"webp"},
ImageMap: map[string]string{},
ExhaustPath: "./exhaust",
MetadataPath: "./metadata",
RemoteRawPath: "./remote-raw",

EnableWebP: false,
EnableAVIF: false,
Expand All @@ -111,6 +116,8 @@ func NewWebPConfig() *WebpConfig {
Concurrency: 262144,
DisableKeepalive: false,
CacheTTL: 259200,

MaxCacheSize: 0,
}
}

Expand Down Expand Up @@ -243,10 +250,10 @@ func LoadConfig() {
log.Warnf("WEBP_DISABLE_KEEPALIVE is not a valid boolean, using value in config.json %t", Config.DisableKeepalive)
}
}
if os.Getenv("CACHE_TTL") != "" {
cacheTTL, err := strconv.Atoi(os.Getenv("CACHE_TTL"))
if os.Getenv("WEBP_CACHE_TTL") != "" {
cacheTTL, err := strconv.Atoi(os.Getenv("WEBP_CACHE_TTL"))
if err != nil {
log.Warnf("CACHE_TTL is not a valid integer, using value in config.json %d", Config.CacheTTL)
log.Warnf("WEBP_CACHE_TTL is not a valid integer, using value in config.json %d", Config.CacheTTL)
} else {
Config.CacheTTL = cacheTTL
}
Expand All @@ -258,6 +265,15 @@ func LoadConfig() {
RemoteCache = cache.New(time.Duration(Config.CacheTTL)*time.Minute, 10*time.Minute)
}

if os.Getenv("WEBP_MAX_CACHE_SIZE") != "" {
maxCacheSize, err := strconv.Atoi(os.Getenv("WEBP_MAX_CACHE_SIZE"))
if err != nil {
log.Warnf("WEBP_MAX_CACHE_SIZE is not a valid integer, using value in config.json %d", Config.MaxCacheSize)
} else {
Config.MaxCacheSize = maxCacheSize
}
}

log.Debugln("Config init complete")
log.Debugln("Config", Config)
}
Expand Down
10 changes: 9 additions & 1 deletion encoder/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,18 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E
FailOnError: boolFalse,
NumPages: intMinusOne,
})
if err != nil {
log.Warnf("Can't open source image: %v", err)
return err
}
defer img.Close()

// Pre-process image(auto rotate, resize, etc.)
preProcessImage(img, imageType, extraParams)
err = preProcessImage(img, imageType, extraParams)
if err != nil {
log.Warnf("Can't pre-process source image: %v", err)
return err
}

switch imageType {
case "webp":
Expand Down
17 changes: 9 additions & 8 deletions handler/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,31 +80,32 @@ func fetchRemoteImg(url string, subdir string) config.MetaFile {
// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
// How do we know if the remote img is changed? we're using hash(etag+length)
var etag string
cacheKey := subdir+":"+helper.HashString(url)

cacheKey := subdir + ":" + helper.HashString(url)

if val, found := config.RemoteCache.Get(cacheKey); found {
if etagVal, ok := val.(string); ok {
log.Infof("Using cache for remote addr: %s", url)
etag = etagVal
} else {
config.RemoteCache.Delete(cacheKey)
}
}
}

if etag == "" {
log.Infof("Remote Addr is %s, pinging for info...", url)
etag = pingURL(url)
if etag != "" {
config.RemoteCache.Set(cacheKey, etag, cache.DefaultExpiration)
}
}

metadata := helper.ReadMetadata(url, etag, subdir)
localRawImagePath := path.Join(config.RemoteRaw, subdir, metadata.Id)
localRawImagePath := path.Join(config.Config.RemoteRawPath, subdir, metadata.Id)
localExhaustImagePath := path.Join(config.Config.ExhaustPath, subdir, metadata.Id)

if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) {
cleanProxyCache(path.Join(config.Config.ExhaustPath, subdir, metadata.Id+"*"))
cleanProxyCache(localExhaustImagePath)
if metadata.Checksum != helper.HashString(etag) {
// remote file has changed
log.Info("Remote file changed, updating metadata and fetching image source...")
Expand Down
2 changes: 1 addition & 1 deletion handler/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func Convert(c *fiber.Ctx) error {
// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200

metadata = fetchRemoteImg(realRemoteAddr, targetHostName)
rawImageAbs = path.Join(config.RemoteRaw, targetHostName, metadata.Id)
rawImageAbs = path.Join(config.Config.RemoteRawPath, targetHostName, metadata.Id)
} else {
// not proxyMode, we'll use local path
metadata = helper.ReadMetadata(reqURIwithQuery, "", targetHostName)
Expand Down
4 changes: 2 additions & 2 deletions handler/router_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ func setupParam() {
config.Config.ImgPath = "../pics"
config.Config.ExhaustPath = "../exhaust_test"
config.Config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"}
config.Metadata = "../metadata"
config.RemoteRaw = "../remote-raw"
config.Config.MetadataPath = "../metadata"
config.Config.RemoteRawPath = "../remote-raw"
config.ProxyMode = false
config.Config.EnableWebP = true
config.Config.EnableAVIF = false
Expand Down
5 changes: 1 addition & 4 deletions helper/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,7 @@ func CheckAllowedType(imgFilename string) bool {
}
imgFilenameExtension := strings.ToLower(path.Ext(imgFilename))
imgFilenameExtension = strings.TrimPrefix(imgFilenameExtension, ".") // .jpg -> jpg
if slices.Contains(config.Config.AllowedTypes, imgFilenameExtension) {
return true
}
return false
return slices.Contains(config.Config.AllowedTypes, imgFilenameExtension)
}

func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string, string) {
Expand Down
27 changes: 13 additions & 14 deletions helper/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,24 +33,23 @@ func ReadMetadata(p, etag string, subdir string) config.MetaFile {
var metadata config.MetaFile
var id, _, _ = getId(p)

buf, err := os.ReadFile(path.Join(config.Metadata, subdir, id+".json"))
if err != nil {
log.Warnf("can't read metadata: %s", err)
WriteMetadata(p, etag, subdir)
return ReadMetadata(p, etag, subdir)
}

err = json.Unmarshal(buf, &metadata)
if err != nil {
log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err)
if buf, err := os.ReadFile(path.Join(config.Config.MetadataPath, subdir, id+".json")); err != nil {
// First time reading metadata, create one
WriteMetadata(p, etag, subdir)
return ReadMetadata(p, etag, subdir)
} else {
err = json.Unmarshal(buf, &metadata)
if err != nil {
log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err)
WriteMetadata(p, etag, subdir)
return ReadMetadata(p, etag, subdir)
}
return metadata
}
return metadata
}

func WriteMetadata(p, etag string, subdir string) config.MetaFile {
_ = os.MkdirAll(path.Join(config.Metadata, subdir), 0755)
_ = os.MkdirAll(path.Join(config.Config.MetadataPath, subdir), 0755)

var id, filepath, sant = getId(p)

Expand All @@ -67,13 +66,13 @@ func WriteMetadata(p, etag string, subdir string) config.MetaFile {
}

buf, _ := json.Marshal(data)
_ = os.WriteFile(path.Join(config.Metadata, subdir, data.Id+".json"), buf, 0644)
_ = os.WriteFile(path.Join(config.Config.MetadataPath, subdir, data.Id+".json"), buf, 0644)
return data
}

func DeleteMetadata(p string, subdir string) {
var id, _, _ = getId(p)
metadataPath := path.Join(config.Metadata, subdir, id+".json")
metadataPath := path.Join(config.Config.MetadataPath, subdir, id+".json")
err := os.Remove(metadataPath)
if err != nil {
log.Warnln("failed to delete metadata", err)
Expand Down
Loading

0 comments on commit 89ea0af

Please sign in to comment.