refactor: query should not return lines being prefixed with the k-pro…

…ximity range in order to mimick upstream API

README.md

@@ -21,11 +21,11 @@ The API is really simple; one type, holding three methods, is exported (and addi
 ```go
 New(options ...CommonOption) *HIBP
 HIBP#Sync(options ...SyncOption) error // Syncs the local copy with the upstream database
-HIBP#Export(w io.Writer, options ...ExportOption) error // Writes a continuous, decompressed and "free-of-etags" stream to the given io.Writer
-HIBP#.Query("ABCDE") (io.ReadClose, error) // Returns the k-proximity API result as the upstream API would
+HIBP#Export(w io.Writer, options ...ExportOption) error // Writes a continuous, decompressed and "free-of-etags" stream to the given io.Writer with the lines being prefix by the k-proximity range
+HIBP#.Query("ABCDE") (io.ReadClose, error) // Returns the k-proximity API result as the upstream API would (without the k-proximity range as prefix)
 ```
 
-All operates operate on disk but, depending on the medium, should provide access times that are probably good enough for all scenarios.
+All of them operate on disk but, depending on the medium, should provide access times that are probably good enough for all scenarios.
 A memory-based `tmpfs` will speed things up when necessary.
 
 

cmd/sync/main.go

@@ -69,7 +69,7 @@ func run(dataDir string) error {
 		return nil
 	}
 
-	h := hibp.New(hibp.WithDataDir(dataDir))
+	h := hibp.New(hibp.WithDataDir(dataDir), hibp.WithNoCompression())
 
 	if err := h.Sync(
 		hibp.SyncWithProgressFn(updateProgressBar),

export.go

@@ -1,6 +1,8 @@
 package hibp
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 )
@@ -18,7 +20,17 @@ func export(from, to int64, store storage, w io.Writer) error {
 			}
 			defer dataReader.Close()
 
-			if _, err := io.Copy(w, dataReader); err != nil {
+			lines, err := io.ReadAll(dataReader)
+			if err != nil {
+				return fmt.Errorf("reading data for range %q: %w", rangePrefix, err)
+			}
+
+			prefixedLines, err := prefixLines(lines, rangePrefix)
+			if err != nil {
+				return fmt.Errorf("prefixing lines for range %q: %w", rangePrefix, err)
+			}
+
+			if _, err := w.Write(prefixedLines); err != nil {
 				return fmt.Errorf("writing data for range %q: %w", rangePrefix, err)
 			}
 
@@ -42,3 +54,32 @@ func export(from, to int64, store storage, w io.Writer) error {
 
 	return nil
 }
+
+func prefixLines(in []byte, prefix string) ([]byte, error) {
+	firstLine := true
+
+	// Actually, we know that the size will be: len(in) + rows * len(prefix)
+	// But we do not know the number of rows - so starting from len(in) seems to be a good choice.
+	out := bytes.NewBuffer(make([]byte, 0, len(in)))
+
+	scanner := bufio.NewScanner(bytes.NewReader(in))
+	for scanner.Scan() {
+		if !firstLine {
+			if _, err := out.Write(lineSeparator); err != nil {
+				return nil, fmt.Errorf("adding line separator: %w", err)
+			}
+		}
+
+		firstLine = false
+
+		if _, err := out.Write([]byte(prefix)); err != nil {
+			return nil, fmt.Errorf("adding prefix: %w", err)
+		}
+
+		if _, err := out.Write(scanner.Bytes()); err != nil {
+			return nil, fmt.Errorf("adding suffix and counter: %w", err)
+		}
+	}
+
+	return out.Bytes(), nil
+}

export_test.go

@@ -12,16 +12,20 @@ func TestExport(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	storageMock := NewMockstorage(ctrl)
 
-	storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("00000suffix:counter11\n00000suffix:counter12"))), nil)
-	storageMock.EXPECT().LoadData("00001").Return(io.NopCloser(bytes.NewReader([]byte("00001suffix:counter2"))), nil)
-	storageMock.EXPECT().LoadData("00002").Return(io.NopCloser(bytes.NewReader([]byte("00002suffix:counter3"))), nil)
+	storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil)
+	storageMock.EXPECT().LoadData("00001").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter2"))), nil)
+	storageMock.EXPECT().LoadData("00002").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter3"))), nil)
 
 	buf := bytes.NewBuffer([]byte{})
 
 	if err := export(0, 3, storageMock, buf); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
+	// We expect the lines to be prefixed with the range as this is what the response from the official
+	// HIBP API looks like.
+	// This has to be the case because `Export` iterates over all ranges; different from `Query` which only
+	// queries a single range.
 	if buf.String() != "00000suffix:counter11\n00000suffix:counter12\n00001suffix:counter2\n00002suffix:counter3" {
 		t.Fatalf("unexpected output: %q", buf.String())
 	}

lib.go

@@ -91,7 +91,9 @@ func (h *HIBP) Sync(options ...SyncOption) error {
 }
 
 // Export writes the dataset to the given writer.
-// The data is written in the same format as it is provided by the Have-I-Been-Pwned API itself.
+// The data is written as a continuous stream with no indication of the "prefix boundaries",
+// the format therefore differs from the official Have-I-Been-Pwned API and from `Query`, which is mimicking the API.
+// Lines have the schema "<prefix><suffix>:<count>".
 func (h *HIBP) Export(w io.Writer) error {
 	return export(0, defaultLastRange+1, h.store, w)
 }
@@ -100,6 +102,8 @@ func (h *HIBP) Export(w io.Writer) error {
 // The function returns an io.ReadCloser that can be used to read the data, it should be closed as soon as possible
 // to release the read lock on the file.
 // It is the responsibility of the caller to close the returned io.ReadCloser.
+// The resulting lines do NOT start with the prefix, they are following the schema "<suffix>:<count>".
+// This is equivalent to the response of the official Have-I-Been-Pwned API.
 func (h *HIBP) Query(prefix string) (io.ReadCloser, error) {
 	reader, err := h.store.LoadData(prefix)
 	if err != nil {

lib_test.go

@@ -1,11 +1,39 @@
 package hibp
 
 import (
+	"bytes"
+	"go.uber.org/mock/gomock"
 	"io"
 	"math/rand"
 	"testing"
 )
 
+func TestQuery(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	storageMock := NewMockstorage(ctrl)
+
+	storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil)
+
+	i := HIBP{store: storageMock}
+
+	reader, err := i.Query("00000")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	defer reader.Close()
+
+	lines, err := io.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	// We expect the lines to not be prefixed with the range as this is what the response from the official
+	// HIBP API looks like.
+	if string(lines) != "suffix:counter11\nsuffix:counter12" {
+		t.Fatalf("unexpected output: %q", string(lines))
+	}
+}
+
 func BenchmarkQuery(b *testing.B) {
 	const lastRange = 0x0000A
 

sync.go

@@ -1,8 +1,6 @@
 package hibp
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -51,12 +49,7 @@ func sync(ctx context.Context, from, to int64, client *hibpClient, store storage
 				}
 
 				if !resp.NotModified {
-					prefixedLines, err := prefixLines(resp.Data, rangePrefix)
-					if err != nil {
-						return fmt.Errorf("prefixing lines: %w", err)
-					}
-
-					if err := store.Save(rangePrefix, resp.ETag, prefixedLines); err != nil {
+					if err := store.Save(rangePrefix, resp.ETag, resp.Data); err != nil {
 						return fmt.Errorf("saving range: %w", err)
 					}
 				}
@@ -98,35 +91,6 @@ func toRangeString(i int64) string {
 	return fmt.Sprintf("%05X", i)
 }
 
-func prefixLines(in []byte, prefix string) ([]byte, error) {
-	firstLine := true
-
-	// Actually, we know that the size will be: len(in) + rows * len(prefix)
-	// But we do not know the number of rows - so starting from len(in) seems to be a good choice.
-	out := bytes.NewBuffer(make([]byte, 0, len(in)))
-
-	scanner := bufio.NewScanner(bytes.NewReader(in))
-	for scanner.Scan() {
-		if !firstLine {
-			if _, err := out.Write(lineSeparator); err != nil {
-				return nil, fmt.Errorf("adding line separator: %w", err)
-			}
-		}
-
-		firstLine = false
-
-		if _, err := out.Write([]byte(prefix)); err != nil {
-			return nil, fmt.Errorf("adding prefix: %w", err)
-		}
-
-		if _, err := out.Write(scanner.Bytes()); err != nil {
-			return nil, fmt.Errorf("adding suffix and counter: %w", err)
-		}
-	}
-
-	return out.Bytes(), nil
-}
-
 func lowestInFlight(inFlight mapset.Set[int64], to int64) int64 {
 	lowest := int64(math.MaxInt64)