Fix/add fee provision 2

db/migrations/20230703120000_add_tx_entry_type.up.sql

@@ -0,0 +1,2 @@
+alter table transaction_entries 
+add column entry_type character varying;

db/migrations/20230703130000_replace_tx_entry_constraint.up.sql

@@ -0,0 +1,2 @@
+alter table transaction_entries drop constraint unique_tx_entry_tuple,
+add constraint unique_tx_entry_tuple UNIQUE(user_id, invoice_id, debit_account_id, credit_account_id, entry_type);

db/models/transactionentry.go

@@ -4,6 +4,15 @@ import (
 	"time"
 )
 
+const (
+	EntryTypeIncoming           = "incoming"
+	EntryTypeOutgoing           = "outgoing"
+	EntryTypeFee                = "fee"
+	EntryTypeFeeReserve         = "fee_reserve"
+	EntryTypeFeeReserveReversal = "fee_reserve_reversal"
+	EntryTypeOutgoingReversal   = "outgoing_reversal"
+)
+
 // TransactionEntry : Transaction Entries Model
 type TransactionEntry struct {
 	ID              int64             `bun:",pk,autoincrement"`
@@ -14,9 +23,11 @@ type TransactionEntry struct {
 	ParentID        int64             `bun:",nullzero"`
 	Parent          *TransactionEntry `bun:"rel:belongs-to"`
 	CreditAccountID int64             `bun:",notnull"`
+	FeeReserve      *TransactionEntry `bun:"rel:belongs-to"`
 	CreditAccount   *Account          `bun:"rel:belongs-to,join:credit_account_id=id"`
 	DebitAccountID  int64             `bun:",notnull"`
 	DebitAccount    *Account          `bun:"rel:belongs-to,join:debit_account_id=id"`
 	Amount          int64             `bun:",notnull"`
 	CreatedAt       time.Time         `bun:",nullzero,notnull,default:current_timestamp"`
+	EntryType       string
 }

integration_tests/hodl_invoice_test.go

@@ -28,13 +28,14 @@ type HodlInvoiceSuite struct {
 	service                  *service.LndhubService
 	userLogin                ExpectedCreateUserResponseBody
 	userToken                string
+	userToken2               string
 	invoiceUpdateSubCancelFn context.CancelFunc
 	hodlLND                  *LNDMockHodlWrapperAsync
 }
 
 func (suite *HodlInvoiceSuite) SetupSuite() {
 	mlnd := newDefaultMockLND()
-	externalLND, err := NewMockLND("1234567890abcdefabcd", 0, make(chan (*lnrpc.Invoice)))
+	externalLND, err := NewMockLND("1234567890abcdefabcd", 0, make(chan (*lnrpc.Invoice), 5))
 	if err != nil {
 		log.Fatalf("Error initializing test service: %v", err)
 	}
@@ -51,7 +52,7 @@ func (suite *HodlInvoiceSuite) SetupSuite() {
 	if err != nil {
 		log.Fatalf("Error initializing test service: %v", err)
 	}
-	users, userTokens, err := createUsers(svc, 1)
+	users, userTokens, err := createUsers(svc, 2)
 	if err != nil {
 		log.Fatalf("Error creating test users: %v", err)
 	}
@@ -66,21 +67,22 @@ func (suite *HodlInvoiceSuite) SetupSuite() {
 	e.HTTPErrorHandler = responses.HTTPErrorHandler
 	e.Validator = &lib.CustomValidator{Validator: validator.New()}
 	suite.echo = e
-	assert.Equal(suite.T(), 1, len(users))
-	assert.Equal(suite.T(), 1, len(userTokens))
+	assert.Equal(suite.T(), 2, len(users))
+	assert.Equal(suite.T(), 2, len(userTokens))
 	suite.userLogin = users[0]
 	suite.userToken = userTokens[0]
+	suite.userToken2 = userTokens[1]
 	suite.echo.Use(tokens.Middleware([]byte(suite.service.Config.JWTSecret)))
 	suite.echo.GET("/balance", controllers.NewBalanceController(suite.service).Balance)
 	suite.echo.POST("/addinvoice", controllers.NewAddInvoiceController(suite.service).AddInvoice)
 	suite.echo.POST("/payinvoice", controllers.NewPayInvoiceController(suite.service).PayInvoice)
 }
 
 func (suite *HodlInvoiceSuite) TestHodlInvoice() {
-	userFundingSats := 1000
-	externalSatRequested := 500
+	userFundingSats := int64(1000)
+	externalSatRequested := int64(500)
 	// fund user account
-	invoiceResponse := suite.createAddInvoiceReq(userFundingSats, "integration test external payment user", suite.userToken)
+	invoiceResponse := suite.createAddInvoiceReq(int(userFundingSats), "integration test external payment user", suite.userToken)
 	err := suite.mlnd.mockPaidInvoice(invoiceResponse, 0, false, nil)
 	assert.NoError(suite.T(), err)
 
@@ -90,7 +92,7 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
 	// create external invoice
 	externalInvoice := lnrpc.Invoice{
 		Memo:      "integration tests: external pay from user",
-		Value:     int64(externalSatRequested),
+		Value:     externalSatRequested,
 		RPreimage: []byte("preimage1"),
 	}
 	invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice)
@@ -107,7 +109,10 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
 	if err != nil {
 		fmt.Printf("Error when getting balance %v\n", err.Error())
 	}
-	assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance)
+
+	//also check that the fee reserve was reduced
+	feeReserve := suite.service.CalcFeeLimit(suite.externalLND.GetMainPubkey(), int64(externalSatRequested))
+	assert.Equal(suite.T(), userFundingSats-externalSatRequested-feeReserve, userBalance)
 
 	// check payment is pending
 	inv, err := suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash))
@@ -153,16 +158,23 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
 	errorString := "FAILURE_REASON_INCORRECT_PAYMENT_DETAILS"
 	assert.Equal(suite.T(), errorString, invoices[0].ErrorMessage)
 
-	transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
+	transactionEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
 	if err != nil {
 		fmt.Printf("Error when getting transaction entries %v\n", err.Error())
 	}
-	// check if there are 3 transaction entries, with reversed credit and debit account ids
-	assert.Equal(suite.T(), 3, len(transactonEntries))
-	assert.Equal(suite.T(), transactonEntries[1].CreditAccountID, transactonEntries[2].DebitAccountID)
-	assert.Equal(suite.T(), transactonEntries[1].DebitAccountID, transactonEntries[2].CreditAccountID)
-	assert.Equal(suite.T(), transactonEntries[1].Amount, int64(externalSatRequested))
-	assert.Equal(suite.T(), transactonEntries[2].Amount, int64(externalSatRequested))
+	// check if there are 5 transaction entries:
+	//	- the incoming payment
+	//  - the outgoing payment
+	//  - the fee reserve + the fee reserve reversal
+	//  - the outgoing payment reversal
+	//  with reversed credit and debit account ids for payment 2/5 & payment 3/4
+	assert.Equal(suite.T(), 5, len(transactionEntries))
+	assert.Equal(suite.T(), transactionEntries[1].CreditAccountID, transactionEntries[4].DebitAccountID)
+	assert.Equal(suite.T(), transactionEntries[1].DebitAccountID, transactionEntries[4].CreditAccountID)
+	assert.Equal(suite.T(), transactionEntries[2].CreditAccountID, transactionEntries[3].DebitAccountID)
+	assert.Equal(suite.T(), transactionEntries[2].DebitAccountID, transactionEntries[3].CreditAccountID)
+	assert.Equal(suite.T(), transactionEntries[1].Amount, int64(externalSatRequested))
+	assert.Equal(suite.T(), transactionEntries[4].Amount, int64(externalSatRequested))
 
 	// create external invoice
 	externalInvoice = lnrpc.Invoice{
@@ -214,12 +226,97 @@ func (suite *HodlInvoiceSuite) TestHodlInvoice() {
 	inv, err = suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash))
 	assert.NoError(suite.T(), err)
 	assert.Equal(suite.T(), common.InvoiceStateSettled, inv.State)
-	clearTable(suite.service, "invoices")
-	clearTable(suite.service, "transaction_entries")
-	clearTable(suite.service, "accounts")
+}
+func (suite *HodlInvoiceSuite) TestNegativeBalanceWithHodl() {
+	//10M funding, 5M sat requested
+	userFundingSats := 10000000
+	externalSatRequested := 5000000
+	userId := getUserIdFromToken(suite.userToken2)
+	// fund user account
+	invoiceResponse := suite.createAddInvoiceReq(userFundingSats, "integration test external payment user", suite.userToken2)
+	err := suite.mlnd.mockPaidInvoice(invoiceResponse, 0, false, nil)
+	assert.NoError(suite.T(), err)
+
+	// wait a bit for the callback event to hit
+	time.Sleep(10 * time.Millisecond)
+
+	// create external invoice
+	externalInvoice := lnrpc.Invoice{
+		Memo:      "integration tests: external pay from user",
+		Value:     int64(externalSatRequested),
+		RPreimage: []byte("preimage3"),
+	}
+	invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice)
+	assert.NoError(suite.T(), err)
+	//the fee should be 1 %, so 50k sats (+1)
+	feeLimit := suite.service.CalcFeeLimit(suite.externalLND.GetMainPubkey(), int64(externalSatRequested))
+	// pay external from user, req will be canceled after 2 sec
+	go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken2)
+	// wait for payment to be updated as pending in database
+	time.Sleep(3 * time.Second)
+	// check payment is pending
+	inv, err := suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash))
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State)
+
+	//drain balance from account: at this point we have 5 M, so we can pay 4.95M
+	drainInv := lnrpc.Invoice{
+		Memo:      "integration tests: external pay from user",
+		Value:     int64(4950000),
+		RPreimage: []byte("preimage4"),
+	}
+	drainInvoice, err := suite.externalLND.AddInvoice(context.Background(), &drainInv)
+	assert.NoError(suite.T(), err)
+	//pay drain invoice
+	go suite.createPayInvoiceReqWithCancel(drainInvoice.PaymentRequest, suite.userToken2)
+	time.Sleep(3 * time.Second)
+
+	//start payment checking loop
+	go func() {
+		err = suite.service.CheckAllPendingOutgoingPayments(context.Background())
+		assert.NoError(suite.T(), err)
+	}()
+	//wait a bit for routine to start
+	time.Sleep(time.Second)
+	//now settle both invoices with the maximum fee
+	suite.hodlLND.SettlePayment(lnrpc.Payment{
+		PaymentHash:     hex.EncodeToString(drainInvoice.RHash),
+		Value:           drainInv.Value,
+		CreationDate:    0,
+		FeeSat:          feeLimit,
+		PaymentPreimage: "preimage3",
+		ValueSat:        drainInv.Value,
+		ValueMsat:       0,
+		PaymentRequest:  invoice.PaymentRequest,
+		Status:          lnrpc.Payment_SUCCEEDED,
+		FailureReason:   0,
+	})
+	//wait a bit for db update to happen
+	time.Sleep(time.Second)
+	//send settle invoice with lnrpc.payment
+	suite.hodlLND.SettlePayment(lnrpc.Payment{
+		PaymentHash:     hex.EncodeToString(invoice.RHash),
+		Value:           externalInvoice.Value,
+		CreationDate:    0,
+		FeeSat:          feeLimit,
+		PaymentPreimage: "preimage4",
+		ValueSat:        externalInvoice.Value,
+		ValueMsat:       0,
+		PaymentRequest:  invoice.PaymentRequest,
+		Status:          lnrpc.Payment_SUCCEEDED,
+		FailureReason:   0,
+	})
+	//fetch user balance again
+	userBalance, err := suite.service.CurrentUserBalance(context.Background(), userId)
+	assert.NoError(suite.T(), err)
+	//assert the balance did not go below 0
+	assert.False(suite.T(), userBalance < 0)
 }
 
 func (suite *HodlInvoiceSuite) TearDownSuite() {
+	clearTable(suite.service, "invoices")
+	clearTable(suite.service, "transaction_entries")
+	clearTable(suite.service, "accounts")
 	suite.invoiceUpdateSubCancelFn()
 }
 

integration_tests/internal_payment_test.go

@@ -171,13 +171,11 @@ func (suite *PaymentTestSuite) TestInternalPayment() {
 	suite.echo.ServeHTTP(rec, req)
 	assert.Equal(suite.T(), http.StatusBadRequest, rec.Code)
 
-	transactonEntriesAlice, _ := suite.service.TransactionEntriesFor(context.Background(), aliceId)
+	transactionEntriesAlice, _ := suite.service.TransactionEntriesFor(context.Background(), aliceId)
 	aliceBalance, _ := suite.service.CurrentUserBalance(context.Background(), aliceId)
-	assert.Equal(suite.T(), 3, len(transactonEntriesAlice))
-	assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntriesAlice[0].Amount)
-	assert.Equal(suite.T(), int64(bobSatRequested), transactonEntriesAlice[1].Amount)
-	assert.Equal(suite.T(), int64(fee), transactonEntriesAlice[2].Amount)
-	assert.Equal(suite.T(), transactonEntriesAlice[1].ID, transactonEntriesAlice[2].ParentID)
+	assert.Equal(suite.T(), 2, len(transactionEntriesAlice))
+	assert.Equal(suite.T(), int64(aliceFundingSats), transactionEntriesAlice[0].Amount)
+	assert.Equal(suite.T(), int64(bobSatRequested), transactionEntriesAlice[1].Amount)
 	assert.Equal(suite.T(), int64(aliceFundingSats-bobSatRequested-fee), aliceBalance)
 
 	bobBalance, _ := suite.service.CurrentUserBalance(context.Background(), bobId)
@@ -241,7 +239,7 @@ func (suite *PaymentTestSuite) TestInternalPaymentFail() {
 	assert.Equal(suite.T(), 2, len(invoices))
 	assert.Equal(suite.T(), common.InvoiceStateError, invoices[0].State)
 	assert.Equal(suite.T(), common.InvoiceStateSettled, invoices[1].State)
-	transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
+	transactionEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
 	if err != nil {
 		fmt.Printf("Error when getting transaction entries %v\n", err.Error())
 	}
@@ -251,15 +249,14 @@ func (suite *PaymentTestSuite) TestInternalPaymentFail() {
 		fmt.Printf("Error when getting balance %v\n", err.Error())
 	}
 
-	// check if there are 5 transaction entries, with reversed credit and debit account ids for last 2
-	assert.Equal(suite.T(), 5, len(transactonEntries))
-	assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntries[0].Amount)
-	assert.Equal(suite.T(), int64(bobSatRequested), transactonEntries[1].Amount)
-	assert.Equal(suite.T(), int64(fee), transactonEntries[2].Amount)
-	assert.Equal(suite.T(), transactonEntries[3].CreditAccountID, transactonEntries[4].DebitAccountID)
-	assert.Equal(suite.T(), transactonEntries[3].DebitAccountID, transactonEntries[4].CreditAccountID)
-	assert.Equal(suite.T(), transactonEntries[3].Amount, int64(bobSatRequested))
-	assert.Equal(suite.T(), transactonEntries[4].Amount, int64(bobSatRequested))
+	// check if there are 4 transaction entries, with reversed credit and debit account ids for last 2
+	assert.Equal(suite.T(), 4, len(transactionEntries))
+	assert.Equal(suite.T(), int64(aliceFundingSats), transactionEntries[0].Amount)
+	assert.Equal(suite.T(), int64(bobSatRequested), transactionEntries[1].Amount)
+	assert.Equal(suite.T(), transactionEntries[2].CreditAccountID, transactionEntries[3].DebitAccountID)
+	assert.Equal(suite.T(), transactionEntries[2].DebitAccountID, transactionEntries[3].CreditAccountID)
+	assert.Equal(suite.T(), transactionEntries[2].Amount, int64(bobSatRequested))
+	assert.Equal(suite.T(), transactionEntries[3].Amount, int64(bobSatRequested))
 	// assert that balance was reduced only once
 	assert.Equal(suite.T(), int64(aliceFundingSats)-int64(bobSatRequested+fee), int64(aliceBalance))
 }

integration_tests/lnd_mock_hodl.go

@@ -33,7 +33,7 @@ func (hps *HodlPaymentSubscriber) Recv() (*lnrpc.Payment, error) {
 func NewLNDMockHodlWrapperAsync(lnd lnd.LightningClientWrapper) (result *LNDMockHodlWrapperAsync, err error) {
 	return &LNDMockHodlWrapperAsync{
 		hps: &HodlPaymentSubscriber{
-			ch: make(chan lnrpc.Payment),
+			ch: make(chan lnrpc.Payment, 5),
 		},
 		LightningClientWrapper: lnd,
 	}, nil

integration_tests/outgoing_payment_test.go

@@ -63,7 +63,7 @@ func (suite *PaymentTestSuite) TestOutGoingPayment() {
 	assert.Equal(suite.T(), 1, len(outgoingInvoices))
 	assert.Equal(suite.T(), 1, len(incomingInvoices))
 
-	assert.Equal(suite.T(), 3, len(transactonEntries))
+	assert.Equal(suite.T(), 5, len(transactonEntries))
 
 	assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntries[0].Amount)
 	assert.Equal(suite.T(), currentAccount.ID, transactonEntries[0].CreditAccountID)
@@ -77,13 +77,13 @@ func (suite *PaymentTestSuite) TestOutGoingPayment() {
 	assert.Equal(suite.T(), int64(0), transactonEntries[1].ParentID)
 	assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[1].InvoiceID)
 
-	assert.Equal(suite.T(), int64(suite.mlnd.fee), transactonEntries[2].Amount)
+	assert.Equal(suite.T(), int64(suite.mlnd.fee), transactonEntries[4].Amount)
 	assert.Equal(suite.T(), feeAccount.ID, transactonEntries[2].CreditAccountID)
 	assert.Equal(suite.T(), currentAccount.ID, transactonEntries[2].DebitAccountID)
 	assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[2].InvoiceID)
 
 	// make sure fee entry parent id is previous entry
-	assert.Equal(suite.T(), transactonEntries[1].ID, transactonEntries[2].ParentID)
+	assert.Equal(suite.T(), transactonEntries[1].ID, transactonEntries[4].ParentID)
 
 	//fetch transactions, make sure the fee is there
 	// check invoices again
@@ -134,7 +134,7 @@ func (suite *PaymentTestSuite) TestOutGoingPaymentWithNegativeBalance() {
 	assert.Equal(suite.T(), int64(-1), aliceBalance)
 
 	// check that no additional transaction entry was created
-	transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
+	transactionEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId)
 	if err != nil {
 		fmt.Printf("Error when getting transaction entries %v\n", err.Error())
 	}
@@ -149,27 +149,27 @@ func (suite *PaymentTestSuite) TestOutGoingPaymentWithNegativeBalance() {
 	assert.Equal(suite.T(), 1, len(outgoingInvoices))
 	assert.Equal(suite.T(), 1, len(incomingInvoices))
 
-	assert.Equal(suite.T(), 3, len(transactonEntries))
+	assert.Equal(suite.T(), 5, len(transactionEntries))
 
-	assert.Equal(suite.T(), int64(aliceFundingSats), transactonEntries[0].Amount)
-	assert.Equal(suite.T(), currentAccount.ID, transactonEntries[0].CreditAccountID)
-	assert.Equal(suite.T(), incomingAccount.ID, transactonEntries[0].DebitAccountID)
-	assert.Equal(suite.T(), int64(0), transactonEntries[0].ParentID)
-	assert.Equal(suite.T(), incomingInvoices[0].ID, transactonEntries[0].InvoiceID)
+	assert.Equal(suite.T(), int64(aliceFundingSats), transactionEntries[0].Amount)
+	assert.Equal(suite.T(), currentAccount.ID, transactionEntries[0].CreditAccountID)
+	assert.Equal(suite.T(), incomingAccount.ID, transactionEntries[0].DebitAccountID)
+	assert.Equal(suite.T(), int64(0), transactionEntries[0].ParentID)
+	assert.Equal(suite.T(), incomingInvoices[0].ID, transactionEntries[0].InvoiceID)
 
-	assert.Equal(suite.T(), int64(externalSatRequested), transactonEntries[1].Amount)
-	assert.Equal(suite.T(), outgoingAccount.ID, transactonEntries[1].CreditAccountID)
-	assert.Equal(suite.T(), currentAccount.ID, transactonEntries[1].DebitAccountID)
-	assert.Equal(suite.T(), int64(0), transactonEntries[1].ParentID)
-	assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[1].InvoiceID)
+	assert.Equal(suite.T(), int64(externalSatRequested), transactionEntries[1].Amount)
+	assert.Equal(suite.T(), outgoingAccount.ID, transactionEntries[1].CreditAccountID)
+	assert.Equal(suite.T(), currentAccount.ID, transactionEntries[1].DebitAccountID)
+	assert.Equal(suite.T(), int64(0), transactionEntries[1].ParentID)
+	assert.Equal(suite.T(), outgoingInvoices[0].ID, transactionEntries[1].InvoiceID)
 
-	assert.Equal(suite.T(), int64(suite.mlnd.fee), transactonEntries[2].Amount)
-	assert.Equal(suite.T(), feeAccount.ID, transactonEntries[2].CreditAccountID)
-	assert.Equal(suite.T(), currentAccount.ID, transactonEntries[2].DebitAccountID)
-	assert.Equal(suite.T(), outgoingInvoices[0].ID, transactonEntries[2].InvoiceID)
+	assert.Equal(suite.T(), int64(suite.mlnd.fee), transactionEntries[4].Amount)
+	assert.Equal(suite.T(), feeAccount.ID, transactionEntries[2].CreditAccountID)
+	assert.Equal(suite.T(), currentAccount.ID, transactionEntries[2].DebitAccountID)
+	assert.Equal(suite.T(), outgoingInvoices[0].ID, transactionEntries[2].InvoiceID)
 
 	// make sure fee entry parent id is previous entry
-	assert.Equal(suite.T(), transactonEntries[1].ID, transactonEntries[2].ParentID)
+	assert.Equal(suite.T(), transactionEntries[1].ID, transactionEntries[4].ParentID)
 }
 
 func (suite *PaymentTestSuite) TestZeroAmountInvoice() {