Add MAX_CACHE_SIZE config (#344)

* Add MAX_CACHE_SIZE

* Bump actions version

* Optimize code

* Use Ticker to control GC loop

.github/workflows/CI.yaml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: '1.22'
 
@@ -35,7 +35,7 @@ jobs:
       pull-requests: write 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 

.github/workflows/codecov.yaml

@@ -14,10 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: '1.22'
 

.github/workflows/codeql-analysis.yml

@@ -35,10 +35,10 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: '1.22'
 

.github/workflows/integration-test.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build image
         uses: docker/build-push-action@v3
@@ -43,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build image
         uses: docker/build-push-action@v3

.github/workflows/release_binary.yaml

@@ -15,13 +15,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Make WebP Server Go (amd64)
         run: |

.github/workflows/release_docker_image.yaml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 

Makefile

@@ -26,7 +26,7 @@ tools-dir:
 
 install-staticcheck: tools-dir
 	GOBIN=`pwd`/tools/bin go install honnef.co/go/tools/cmd/staticcheck@latest
-	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.52.2
+	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.59.1
 
 static-check: install-staticcheck
 	#S1000,SA1015,SA4006,SA4011,S1023,S1034,ST1003,ST1005,ST1016,ST1020,ST1021
@@ -39,6 +39,5 @@ test:
 clean:
 	rm -rf prefetch remote-raw exhaust tools coverage.txt metadata exhaust_test
 
-
 docker:
 	DOCKER_BUILDKIT=1 docker build -t webpsh/webps .

config/config.go

@@ -37,7 +37,8 @@ const (
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
   "DISABLE_KEEPALIVE": false,
-  "CACHE_TTL": 259200
+  "CACHE_TTL": 259200,
+  "MAX_CACHE_SIZE": 0
 }`
 )
 
@@ -50,11 +51,9 @@ var (
 	ProxyMode      bool
 	Prefetch       bool
 	Config         = NewWebPConfig()
-	Version        = "0.11.3"
+	Version        = "0.11.4"
 	WriteLock      = cache.New(5*time.Minute, 10*time.Minute)
 	ConvertLock    = cache.New(5*time.Minute, 10*time.Minute)
-	RemoteRaw      = "./remote-raw"
-	Metadata       = "./metadata"
 	LocalHostAlias = "local"
 	RemoteCache    *cache.Cache
 )
@@ -66,14 +65,16 @@ type MetaFile struct {
 }
 
 type WebpConfig struct {
-	Host         string            `json:"HOST"`
-	Port         string            `json:"PORT"`
-	ImgPath      string            `json:"IMG_PATH"`
-	Quality      int               `json:"QUALITY,string"`
-	AllowedTypes []string          `json:"ALLOWED_TYPES"`
-	ConvertTypes []string          `json:"CONVERT_TYPES"`
-	ImageMap     map[string]string `json:"IMG_MAP"`
-	ExhaustPath  string            `json:"EXHAUST_PATH"`
+	Host          string            `json:"HOST"`
+	Port          string            `json:"PORT"`
+	ImgPath       string            `json:"IMG_PATH"`
+	Quality       int               `json:"QUALITY,string"`
+	AllowedTypes  []string          `json:"ALLOWED_TYPES"`
+	ConvertTypes  []string          `json:"CONVERT_TYPES"`
+	ImageMap      map[string]string `json:"IMG_MAP"`
+	ExhaustPath   string            `json:"EXHAUST_PATH"`
+	MetadataPath  string            `json:"METADATA_PATH"`
+	RemoteRawPath string            `json:"REMOTE_RAW_PATH"`
 
 	EnableWebP bool `json:"ENABLE_WEBP"`
 	EnableAVIF bool `json:"ENABLE_AVIF"`
@@ -86,19 +87,23 @@ type WebpConfig struct {
 	ReadBufferSize   int  `json:"READ_BUFFER_SIZE"`
 	Concurrency      int  `json:"CONCURRENCY"`
 	DisableKeepalive bool `json:"DISABLE_KEEPALIVE"`
-	CacheTTL         int  `json:"CACHE_TTL"`
+	CacheTTL         int  `json:"CACHE_TTL"` // In minutes
+
+	MaxCacheSize int `json:"MAX_CACHE_SIZE"` // In MB, for max cached exhausted/metadata files(plus remote-raw if applicable), 0 means no limit
 }
 
 func NewWebPConfig() *WebpConfig {
 	return &WebpConfig{
-		Host:         "0.0.0.0",
-		Port:         "3333",
-		ImgPath:      "./pics",
-		Quality:      80,
-		AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
-		ConvertTypes: []string{"webp"},
-		ImageMap:     map[string]string{},
-		ExhaustPath:  "./exhaust",
+		Host:          "0.0.0.0",
+		Port:          "3333",
+		ImgPath:       "./pics",
+		Quality:       80,
+		AllowedTypes:  []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
+		ConvertTypes:  []string{"webp"},
+		ImageMap:      map[string]string{},
+		ExhaustPath:   "./exhaust",
+		MetadataPath:  "./metadata",
+		RemoteRawPath: "./remote-raw",
 
 		EnableWebP: false,
 		EnableAVIF: false,
@@ -111,6 +116,8 @@ func NewWebPConfig() *WebpConfig {
 		Concurrency:                262144,
 		DisableKeepalive:           false,
 		CacheTTL:                   259200,
+
+		MaxCacheSize: 0,
 	}
 }
 
@@ -243,10 +250,10 @@ func LoadConfig() {
 			log.Warnf("WEBP_DISABLE_KEEPALIVE is not a valid boolean, using value in config.json %t", Config.DisableKeepalive)
 		}
 	}
-	if os.Getenv("CACHE_TTL") != "" {
-		cacheTTL, err := strconv.Atoi(os.Getenv("CACHE_TTL"))
+	if os.Getenv("WEBP_CACHE_TTL") != "" {
+		cacheTTL, err := strconv.Atoi(os.Getenv("WEBP_CACHE_TTL"))
 		if err != nil {
-			log.Warnf("CACHE_TTL is not a valid integer, using value in config.json %d", Config.CacheTTL)
+			log.Warnf("WEBP_CACHE_TTL is not a valid integer, using value in config.json %d", Config.CacheTTL)
 		} else {
 			Config.CacheTTL = cacheTTL
 		}
@@ -258,6 +265,15 @@ func LoadConfig() {
 		RemoteCache = cache.New(time.Duration(Config.CacheTTL)*time.Minute, 10*time.Minute)
 	}
 
+	if os.Getenv("WEBP_MAX_CACHE_SIZE") != "" {
+		maxCacheSize, err := strconv.Atoi(os.Getenv("WEBP_MAX_CACHE_SIZE"))
+		if err != nil {
+			log.Warnf("WEBP_MAX_CACHE_SIZE is not a valid integer, using value in config.json %d", Config.MaxCacheSize)
+		} else {
+			Config.MaxCacheSize = maxCacheSize
+		}
+	}
+
 	log.Debugln("Config init complete")
 	log.Debugln("Config", Config)
 }

encoder/encoder.go

@@ -126,10 +126,18 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E
 		FailOnError: boolFalse,
 		NumPages:    intMinusOne,
 	})
+	if err != nil {
+		log.Warnf("Can't open source image: %v", err)
+		return err
+	}
 	defer img.Close()
 
 	// Pre-process image(auto rotate, resize, etc.)
-	preProcessImage(img, imageType, extraParams)
+	err = preProcessImage(img, imageType, extraParams)
+	if err != nil {
+		log.Warnf("Can't pre-process source image: %v", err)
+		return err
+	}
 
 	switch imageType {
 	case "webp":

handler/remote.go

@@ -80,31 +80,32 @@ func fetchRemoteImg(url string, subdir string) config.MetaFile {
 	// url is https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 	// How do we know if the remote img is changed? we're using hash(etag+length)
 	var etag string
-	
-	cacheKey := subdir+":"+helper.HashString(url)
-	
+
+	cacheKey := subdir + ":" + helper.HashString(url)
+
 	if val, found := config.RemoteCache.Get(cacheKey); found {
 		if etagVal, ok := val.(string); ok {
 			log.Infof("Using cache for remote addr: %s", url)
 			etag = etagVal
 		} else {
 			config.RemoteCache.Delete(cacheKey)
 		}
-	} 
-	
+	}
+
 	if etag == "" {
 		log.Infof("Remote Addr is %s, pinging for info...", url)
 		etag = pingURL(url)
 		if etag != "" {
 			config.RemoteCache.Set(cacheKey, etag, cache.DefaultExpiration)
 		}
 	}
-	
+
 	metadata := helper.ReadMetadata(url, etag, subdir)
-	localRawImagePath := path.Join(config.RemoteRaw, subdir, metadata.Id)
+	localRawImagePath := path.Join(config.Config.RemoteRawPath, subdir, metadata.Id)
+	localExhaustImagePath := path.Join(config.Config.ExhaustPath, subdir, metadata.Id)
 
 	if !helper.ImageExists(localRawImagePath) || metadata.Checksum != helper.HashString(etag) {
-		cleanProxyCache(path.Join(config.Config.ExhaustPath, subdir, metadata.Id+"*"))
+		cleanProxyCache(localExhaustImagePath)
 		if metadata.Checksum != helper.HashString(etag) {
 			// remote file has changed
 			log.Info("Remote file changed, updating metadata and fetching image source...")

handler/router.go

@@ -123,7 +123,7 @@ func Convert(c *fiber.Ctx) error {
 		// https://test.webp.sh/mypic/123.jpg?someother=200&somebugs=200
 
 		metadata = fetchRemoteImg(realRemoteAddr, targetHostName)
-		rawImageAbs = path.Join(config.RemoteRaw, targetHostName, metadata.Id)
+		rawImageAbs = path.Join(config.Config.RemoteRawPath, targetHostName, metadata.Id)
 	} else {
 		// not proxyMode, we'll use local path
 		metadata = helper.ReadMetadata(reqURIwithQuery, "", targetHostName)

handler/router_test.go

@@ -33,8 +33,8 @@ func setupParam() {
 	config.Config.ImgPath = "../pics"
 	config.Config.ExhaustPath = "../exhaust_test"
 	config.Config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"}
-	config.Metadata = "../metadata"
-	config.RemoteRaw = "../remote-raw"
+	config.Config.MetadataPath = "../metadata"
+	config.Config.RemoteRawPath = "../remote-raw"
 	config.ProxyMode = false
 	config.Config.EnableWebP = true
 	config.Config.EnableAVIF = false

helper/helper.go

@@ -90,10 +90,7 @@ func CheckAllowedType(imgFilename string) bool {
 	}
 	imgFilenameExtension := strings.ToLower(path.Ext(imgFilename))
 	imgFilenameExtension = strings.TrimPrefix(imgFilenameExtension, ".") // .jpg -> jpg
-	if slices.Contains(config.Config.AllowedTypes, imgFilenameExtension) {
-		return true
-	}
-	return false
+	return slices.Contains(config.Config.AllowedTypes, imgFilenameExtension)
 }
 
 func GenOptimizedAbsPath(metadata config.MetaFile, subdir string) (string, string, string) {

helper/metadata.go

@@ -33,24 +33,23 @@ func ReadMetadata(p, etag string, subdir string) config.MetaFile {
 	var metadata config.MetaFile
 	var id, _, _ = getId(p)
 
-	buf, err := os.ReadFile(path.Join(config.Metadata, subdir, id+".json"))
-	if err != nil {
-		log.Warnf("can't read metadata: %s", err)
-		WriteMetadata(p, etag, subdir)
-		return ReadMetadata(p, etag, subdir)
-	}
-
-	err = json.Unmarshal(buf, &metadata)
-	if err != nil {
-		log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err)
+	if buf, err := os.ReadFile(path.Join(config.Config.MetadataPath, subdir, id+".json")); err != nil {
+		// First time reading metadata, create one
 		WriteMetadata(p, etag, subdir)
 		return ReadMetadata(p, etag, subdir)
+	} else {
+		err = json.Unmarshal(buf, &metadata)
+		if err != nil {
+			log.Warnf("unmarshal metadata error, possible corrupt file, re-building...: %s", err)
+			WriteMetadata(p, etag, subdir)
+			return ReadMetadata(p, etag, subdir)
+		}
+		return metadata
 	}
-	return metadata
 }
 
 func WriteMetadata(p, etag string, subdir string) config.MetaFile {
-	_ = os.MkdirAll(path.Join(config.Metadata, subdir), 0755)
+	_ = os.MkdirAll(path.Join(config.Config.MetadataPath, subdir), 0755)
 
 	var id, filepath, sant = getId(p)
 
@@ -67,13 +66,13 @@ func WriteMetadata(p, etag string, subdir string) config.MetaFile {
 	}
 
 	buf, _ := json.Marshal(data)
-	_ = os.WriteFile(path.Join(config.Metadata, subdir, data.Id+".json"), buf, 0644)
+	_ = os.WriteFile(path.Join(config.Config.MetadataPath, subdir, data.Id+".json"), buf, 0644)
 	return data
 }
 
 func DeleteMetadata(p string, subdir string) {
 	var id, _, _ = getId(p)
-	metadataPath := path.Join(config.Metadata, subdir, id+".json")
+	metadataPath := path.Join(config.Config.MetadataPath, subdir, id+".json")
 	err := os.Remove(metadataPath)
 	if err != nil {
 		log.Warnln("failed to delete metadata", err)