fix: normalize files to UTF8 when extracting zip files for ingest

SpecterOps · Sep 19, 2024 · 6e8a0f5 · 6e8a0f5
1 parent 518570b
commit 6e8a0f5
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 7 deletions.
diff --git a/cmd/api/src/daemons/datapipe/jobs.go b/cmd/api/src/daemons/datapipe/jobs.go
@@ -23,6 +23,7 @@ import (
 	"io"
 	"os"
 
+	"github.com/specterops/bloodhound/bomenc"
 	"github.com/specterops/bloodhound/dawgs/graph"
 	"github.com/specterops/bloodhound/dawgs/util"
 	"github.com/specterops/bloodhound/log"
@@ -165,7 +166,10 @@ func (s *Daemon) preProcessIngestFile(path string, fileType model.FileType) ([]s
 			} else if srcFile, err := f.Open(); err != nil {
 				errs.Add(fmt.Errorf("error opening file %s in archive %s: %v", f.Name, path, err))
 				failed++
-			} else if _, err := io.Copy(tempFile, srcFile); err != nil {
+			} else if normFile, err := bomenc.NormalizeToUTF8(srcFile); err != nil {
+				errs.Add(fmt.Errorf("error normalizing file %s to UTF8 in archive %s: %v", f.Name, path, err))
+				failed++
+			} else if _, err := io.Copy(tempFile, normFile); err != nil {
 				errs.Add(fmt.Errorf("error extracting file %s in archive %s: %v", f.Name, path, err))
 				failed++
 			} else if err := tempFile.Close(); err != nil {

diff --git a/cmd/api/src/services/fileupload/file_upload.go b/cmd/api/src/services/fileupload/file_upload.go
@@ -38,12 +38,6 @@ import (
 
 const jobActivityTimeout = time.Minute * 20
 
-const (
-	UTF8BOM1 = 0xef
-	UTF8BOM2 = 0xbb
-	UTF8BMO3 = 0xbf
-)
-
 var ErrInvalidJSON = errors.New("file is not valid json")
 
 type FileUploadData interface {