fix: document that we use CRLF as line ending, same as upstream API d…

…oes, and update tests accordingly

README.md

@@ -28,6 +28,15 @@ HIBP#.Query("ABCDE") (io.ReadClose, error) // Returns the k-proximity API result
 All of them operate on disk but, depending on the medium, should provide access times that are probably good enough for all scenarios.
 A memory-based `tmpfs` will speed things up when necessary.
 
+**Attention:**
+The [official API](https://haveibeenpwned.com/API/v3#PwnedPasswords) states the following regarding the format: 
+
+> Each password is stored as both a SHA-1 and an NTLM hash of a UTF-8 encoded password. 
+> The downloadable source data delimits the hash and the password count with a colon (:) and each line with a CRLF.
+
+The crucial part being that lines are ended with `\r\n`.
+In order to be compatible with the upstream API this library sticks to this...
+
 
 ## CLI
 

cmd/sync/main.go

@@ -69,7 +69,7 @@ func run(dataDir string) error {
 		return nil
 	}
 
-	h := hibp.New(hibp.WithDataDir(dataDir), hibp.WithNoCompression())
+	h := hibp.New(hibp.WithDataDir(dataDir))
 
 	if err := h.Sync(
 		hibp.SyncWithProgressFn(updateProgressBar),

export.go

@@ -7,7 +7,9 @@ import (
 	"io"
 )
 
-var lineSeparator = []byte("\n")
+// The upstream Have-I-Been-Pwned API uses CRLF as line separator - so we are stuck with it,
+// although it does not feel right.
+var lineSeparator = []byte("\r\n")
 
 func export(from, to int64, store storage, w io.Writer) error {
 	for i := from; i < to; i++ {

export_test.go

@@ -12,7 +12,7 @@ func TestExport(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	storageMock := NewMockstorage(ctrl)
 
-	storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil)
+	storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\r\nsuffix:counter12"))), nil)
 	storageMock.EXPECT().LoadData("00001").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter2"))), nil)
 	storageMock.EXPECT().LoadData("00002").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter3"))), nil)
 
@@ -26,7 +26,7 @@ func TestExport(t *testing.T) {
 	// HIBP API looks like.
 	// This has to be the case because `Export` iterates over all ranges; different from `Query` which only
 	// queries a single range.
-	if buf.String() != "00000suffix:counter11\n00000suffix:counter12\n00001suffix:counter2\n00002suffix:counter3" {
+	if buf.String() != "00000suffix:counter11\r\n00000suffix:counter12\r\n00001suffix:counter2\r\n00002suffix:counter3" {
 		t.Fatalf("unexpected output: %q", buf.String())
 	}
 }

lib_test.go

@@ -12,7 +12,7 @@ func TestQuery(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	storageMock := NewMockstorage(ctrl)
 
-	storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil)
+	storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\r\nsuffix:counter12"))), nil)
 
 	i := HIBP{store: storageMock}
 
@@ -29,7 +29,7 @@ func TestQuery(t *testing.T) {
 
 	// We expect the lines to not be prefixed with the range as this is what the response from the official
 	// HIBP API looks like.
-	if string(lines) != "suffix:counter11\nsuffix:counter12" {
+	if string(lines) != "suffix:counter11\r\nsuffix:counter12" {
 		t.Fatalf("unexpected output: %q", string(lines))
 	}
 }

sync_test.go

@@ -23,63 +23,63 @@ func TestSync(t *testing.T) {
 		Get("/range/00000").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix1")
+		BodyString("suffix1:1")
 	gock.New(baseURL).
 		Get("/range/00001").
 		MatchHeader("If-None-Match", "etag received earlier").
 		Reply(http.StatusNotModified).
 		AddHeader("ETag", "etag received earlier").
-		BodyString("suffix2")
+		BodyString("suffix2:2")
 	gock.New(baseURL).
 		Get("/range/00002").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix31:2\nsuffix32:3")
+		BodyString("suffix31:2\r\nsuffix32:3")
 	gock.New(baseURL).
 		Get("/range/00003").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix4")
+		BodyString("suffix4:4")
 	gock.New(baseURL).
 		Get("/range/00004").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix5")
+		BodyString("suffix5:5")
 	gock.New(baseURL).
 		Get("/range/00005").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix6")
+		BodyString("suffix6:6")
 	gock.New(baseURL).
 		Get("/range/00006").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix7")
+		BodyString("suffix7:7")
 	gock.New(baseURL).
 		Get("/range/00007").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix8")
+		BodyString("suffix8:8")
 	gock.New(baseURL).
 		Get("/range/00008").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix9")
+		BodyString("suffix9:9")
 	gock.New(baseURL).
 		Get("/range/00009").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix10")
+		BodyString("suffix10:10")
 	gock.New(baseURL).
 		Get("/range/0000A").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix11")
+		BodyString("suffix11:11")
 	gock.New(baseURL).
 		Get("/range/0000B").
 		Reply(200).
 		AddHeader("ETag", "etag").
-		BodyString("suffix12")
+		BodyString("suffix12:12")
 
 	client := &hibpClient{
 		endpoint:   defaultEndpoint,
@@ -90,29 +90,29 @@ func TestSync(t *testing.T) {
 	storageMock := NewMockstorage(ctrl)
 
 	storageMock.EXPECT().LoadETag("00000").Return("", nil)
-	storageMock.EXPECT().Save("00000", "etag", []byte("suffix1")).Return(nil)
+	storageMock.EXPECT().Save("00000", "etag", []byte("suffix1:1")).Return(nil)
 	storageMock.EXPECT().LoadETag("00001").Return("etag received earlier", nil)
 	// 00001 does not need to be written as its ETag has not changed
 	storageMock.EXPECT().LoadETag("00002").Return("", nil)
-	storageMock.EXPECT().Save("00002", "etag", []byte("suffix31:2\n00suffix32:3")).Return(nil)
+	storageMock.EXPECT().Save("00002", "etag", []byte("suffix31:2\r\nsuffix32:3")).Return(nil)
 	storageMock.EXPECT().LoadETag("00003").Return("", nil)
-	storageMock.EXPECT().Save("00003", "etag", []byte("suffix4")).Return(nil)
+	storageMock.EXPECT().Save("00003", "etag", []byte("suffix4:4")).Return(nil)
 	storageMock.EXPECT().LoadETag("00004").Return("", nil)
-	storageMock.EXPECT().Save("00004", "etag", []byte("suffix5")).Return(nil)
+	storageMock.EXPECT().Save("00004", "etag", []byte("suffix5:5")).Return(nil)
 	storageMock.EXPECT().LoadETag("00005").Return("", nil)
-	storageMock.EXPECT().Save("00005", "etag", []byte("suffix6")).Return(nil)
+	storageMock.EXPECT().Save("00005", "etag", []byte("suffix6:6")).Return(nil)
 	storageMock.EXPECT().LoadETag("00006").Return("", nil)
-	storageMock.EXPECT().Save("00006", "etag", []byte("suffix7")).Return(nil)
+	storageMock.EXPECT().Save("00006", "etag", []byte("suffix7:7")).Return(nil)
 	storageMock.EXPECT().LoadETag("00007").Return("", nil)
-	storageMock.EXPECT().Save("00007", "etag", []byte("suffix8")).Return(nil)
+	storageMock.EXPECT().Save("00007", "etag", []byte("suffix8:8")).Return(nil)
 	storageMock.EXPECT().LoadETag("00008").Return("", nil)
-	storageMock.EXPECT().Save("00008", "etag", []byte("suffix9")).Return(nil)
+	storageMock.EXPECT().Save("00008", "etag", []byte("suffix9:9")).Return(nil)
 	storageMock.EXPECT().LoadETag("00009").Return("", nil)
-	storageMock.EXPECT().Save("00009", "etag", []byte("suffix10")).Return(nil)
+	storageMock.EXPECT().Save("00009", "etag", []byte("suffix10:10")).Return(nil)
 	storageMock.EXPECT().LoadETag("0000A").Return("", nil)
-	storageMock.EXPECT().Save("0000A", "etag", []byte("suffix11")).Return(nil)
+	storageMock.EXPECT().Save("0000A", "etag", []byte("suffix11:11")).Return(nil)
 	storageMock.EXPECT().LoadETag("0000B").Return("", nil)
-	storageMock.EXPECT().Save("0000B", "etag", []byte("suffix12")).Return(nil)
+	storageMock.EXPECT().Save("0000B", "etag", []byte("suffix12:12")).Return(nil)
 
 	var callCounter atomic.Int64