fix: handle different link types and add unit test for raw ip encapsu…

…lated dump file

csv.go

@@ -20,14 +20,13 @@ import (
 	"strings"
 
 	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
 )
 
 // ReadFileCSV reads the PCAP file at the given path
 // and prints out all packets containing JA3 digests to the supplied io.Writer
 func ReadFileCSV(file string, out io.Writer, separator string, doJA3s bool) {
 
-	r, f, err := openPcap(file)
+	r, f, link, err := openPcap(file)
 	if err != nil {
 		panic(err)
 	}
@@ -54,7 +53,7 @@ func ReadFileCSV(file string, out io.Writer, separator string, doJA3s bool) {
 
 		var (
 			// create gopacket
-			p = gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Lazy)
+			p = gopacket.NewPacket(data, link, gopacket.Lazy)
 			// get JA3 if possible
 			digest   = DigestHexPacket(p)
 			isServer bool

go.sum

@@ -1,21 +1,16 @@
 github.com/dreadl0ck/tlsx v1.0.1-google-gopacket h1:/P3y+CGRiCQbW0nZU2jWkEwKfXLkpEgHNhbbqlnrTTM=
 github.com/dreadl0ck/tlsx v1.0.1-google-gopacket/go.mod h1:amAb73WEEgPHWniMfwro6UpN6St3e5ypgq2tXM89IOo=
-github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
 github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
 github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY=
 github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 h1:1Fzlr8kkDLQwqMP8GxrhptBLqZG/EDpiATneiZHY998=
 golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

ja3s.go

@@ -22,17 +22,15 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-
 	"github.com/dreadl0ck/tlsx"
+	"github.com/google/gopacket"
 )
 
 // ReadFileJa3s reads the PCAP file at the given path
 // and prints out all packets containing JA3S digests to the supplied io.Writer
 func ReadFileJa3s(file string, out io.Writer) {
 
-	r, f, err := openPcap(file)
+	r, f, link, err := openPcap(file)
 	if err != nil {
 		panic(err)
 	}
@@ -53,7 +51,7 @@ func ReadFileJa3s(file string, out io.Writer) {
 
 		var (
 			// create gopacket
-			p = gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Lazy)
+			p = gopacket.NewPacket(data, link, gopacket.Lazy)
 			// get JA3 if possible
 			digest = DigestHexPacketJa3s(p)
 		)

ja3s_test.go

@@ -21,9 +21,9 @@ import (
 	"log"
 	"testing"
 
+	"github.com/dreadl0ck/tlsx"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
-	"github.com/dreadl0ck/tlsx"
 )
 
 var tlsServerHelloPacket, errDecodeServerHelloPacket = hex.DecodeString("f018982a38be48d343aac4b80800450005b4447b00007906ea20225f7893c0a8b20d01bbefca6011d3eb8013192d801000f087ae00000101080a79facb3a56d9798616030300640200006003035e5bea4420be035069306e15e36eca187d56a201e8e96f0a1afeeb529413fba020746572b9b8832a5b8f236ba51743281c20e923a880067e4c38a754f53f3a8743c02f00001800170000ff01000100000b0002010000100005000302683216030309980b0009940009910004fc308204f8308203e0a00302010202100a86b904765831e240cc6211101f5736300d06092a864886f70d01010b0500305e310b300906035504061302555331153013060355040a130c446967694365727420496e6331193017060355040b13107777772e64696769636572742e636f6d311d301b0603550403131447656f5472757374205253412043412032303138301e170d3138303130343030303030305a170d3230303730393132303030305a3068310b3009060355040613025553311330110603550408130a43616c69666f726e69613111300f060355040713085061736164656e61311b3019060355040a13124f70656e5820546563686e6f6c6f676965733114301206035504030c0b2a2e6f70656e782e6e657430820122300d06092a864886f70d01010105000382010f003082010a0282010100bf0efabfe55538e687d4db8f8d615ab1d5dce14702307ca21c0547cd2cc43e455627b4d8decd8015296c19127d29025fbac71f2fc1af1bfe525ed0c2778028a3e7dd643b60842801d9d196651924057827c4d3ce6ae62253b869f384d56a72060055e3537a67a9904e61a4628067f07c680a8bf30c14444d090910c182ea3c5e473b1ab1233838645b0bdfb24265ee32e3521b4bdbae6d79acdbdd7a00c0f00479537079fd352aade5e8394be2f2b397366cfd64bb94b1b96515b4e78d3a2fac04e9d5787792fcf411b6b66cb5dfbc4cb023d93fa99cee630d8ed3558b33e0f2a019ecc5eac16919c48a844f693b5e7ccb4eb84724ab963979c570eb7a3b58ad0203010001a38201a6308201a2301f0603551d230418301680149058ffb09c75a8515477b1edf2a34316389e6cc5301d0603551d0e04160414e03952aaaeb23c67e939ec6022188686bcb314c430210603551d11041a3018820b2a2e6f70656e782e6e657482096f70656e782e6e6574300e0603551d0f0101ff0404030205a0301d0603551d250416301406082b0601050507030106082b06010505070302303e0603551d1f043730353033a031a02f862d687474703a2f2f6364702e67656f74727573742e636f6d2f47656f54727573745253414341323031382e63726c304c0603551d2004453043303706096086480186fd6c0101302a302806082b06010505070201161c68747470733a2f2f7777772e64696769636572742e636f6d2f4350533008060667810c010202307506082b0601050507010104693067302606082b06010505073001861a687474703a2f2f7374617475732e67656f74727573742e636f6d303d06082b060105050730028631687474703a2f2f636163657274732e67656f74727573742e636f6d2f47656f54727573745253414341323031382e63727430090603551d1304023000300d06092a864886f70d01010b0500038201010045a0d33fa95d251c861518096f2ffe89bae2e062cfc4505859b42adeaa775fcf34c2eebde155cfb6449a806abffbaad2e88d77b1397e76ea08265dbc2aa98cb04e3d141c27ed629961d9cd90bb7f6db672408bb15126dd4df4cdf831465c1ee9f321505ace6e7c09d864683fbbaa8ee9fc73e74feefaa5a027067856000f56c9ae39bcb3cc586928429bd8abe5d2a3d8df5c63b4db45fa2cc3ffa6f75b946188019e162fb8dfacf2be3064bf52788378e61c8397ec13e72649fbe15e5c5b6d8bae43ca7b188eda5a41e512f508206fefe96a832ce4676a425ddd06e3cfe93046140818a7359723f2477315f38ab51f83c9dced627efaf6541ed3e9c0767376b200048f3082048b30820373a0")
@@ -73,6 +73,19 @@ func TestDigestHexJa3sCorrect(t *testing.T) {
 	}
 }
 
+func TestPcapWithNonEthernetEncapsulation(t *testing.T) {
+	var (
+		b bytes.Buffer
+	)
+
+	// read test2.pcap and generate fingerprints
+	ReadFileJSON("test2.pcap", &b, true)
+
+	if b.String() == "" {
+		t.Fatal("expected output")
+	}
+}
+
 func TestDigestHexJa3sComparePcap(t *testing.T) {
 
 	// Step 1: Read output of reference implementation

json.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
 )
 
 // Record contains all information for a calculated JA3
@@ -43,7 +42,7 @@ type Record struct {
 // and prints out all packets containing JA3 digests formatted as JSON to the supplied io.Writer
 func ReadFileJSON(file string, out io.Writer, doJA3s bool) {
 
-	r, f, err := openPcap(file)
+	r, f, link, err := openPcap(file)
 	if err != nil {
 		panic(err)
 	}
@@ -62,7 +61,7 @@ func ReadFileJSON(file string, out io.Writer, doJA3s bool) {
 
 		var (
 			// create gopacket
-			p = gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Lazy)
+			p = gopacket.NewPacket(data, link, gopacket.Lazy)
 
 			// get JA3 if possible
 			bare     = BarePacket(p)

utils.go

@@ -2,6 +2,7 @@ package ja3
 
 import (
 	"fmt"
+	"github.com/google/gopacket/layers"
 	"os"
 
 	"github.com/google/gopacket"
@@ -13,7 +14,7 @@ type PacketSource interface {
 	ReadPacketData() ([]byte, gopacket.CaptureInfo, error)
 }
 
-func openPcap(file string) (PacketSource, *os.File, error) {
+func openPcap(file string) (PacketSource, *os.File, layers.LinkType, error) {
 
 	// get file handle
 	f, err := os.Open(file)
@@ -22,12 +23,12 @@ func openPcap(file string) (PacketSource, *os.File, error) {
 	}
 
 	var (
-		r       PacketSource
-		errPcap error
+		reader       PacketSource
+		linkType     layers.LinkType
 	)
 
 	// try to create pcap reader
-	r, errPcap = pcapgo.NewReader(f)
+	pcapReader, errPcap := pcapgo.NewReader(f)
 	if errPcap != nil {
 
 		// maybe its a PCAPNG
@@ -42,15 +43,20 @@ func openPcap(file string) (PacketSource, *os.File, error) {
 		}
 
 		// try to create pcapng reader
-		var errPcapNg error
-		r, errPcapNg = pcapgo.NewNgReader(f, pcapgo.DefaultNgReaderOptions)
+		ngReader, errPcapNg := pcapgo.NewNgReader(f, pcapgo.DefaultNgReaderOptions)
 		if errPcapNg != nil {
 			// nope
 			fmt.Println("pcap error:", errPcap)
 			fmt.Println("pcap-ng error:", errPcapNg)
 			panic("cannot open PCAP file")
 		}
+
+		linkType = ngReader.LinkType()
+		reader = ngReader
+	} else {
+		linkType = pcapReader.LinkType()
+		reader = pcapReader
 	}
 
-	return r, f, err
+	return reader, f, linkType, err
 }