Merge branch 'master' into chore.playtomic-feedback

.github/workflows/tests.yaml

@@ -17,6 +17,10 @@ jobs:
       matrix:
         FEATURES: [ oss ,enterprise ]
     steps:
+      - name: Disable IPv6 (temporary fix)
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
       - name: Checkout
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
@@ -63,6 +67,10 @@ jobs:
           - package: warehouse/integrations/snowflake
             destination: snowflake
     steps:
+      - name: Disable IPv6 (temporary fix)
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
@@ -101,6 +109,10 @@ jobs:
     name: Unit
     runs-on: ubuntu-latest
     steps:
+      - name: Disable IPv6 (temporary fix)
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
@@ -135,13 +147,21 @@ jobs:
           - router
           - services
           - services/rsources
+          - services/dedup
           - suppression-backup-service
           - warehouse
         include:
           - package: services
             exclude: services/rsources
+          - package: services
+            exclude: services/dedup
 
     steps:
+      - name: Disable IPv6 (temporary fix)
+        if: matrix.package != 'services/dedup'
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [1.34.1](https://github.com/rudderlabs/rudder-server/compare/v1.34.0...v1.34.1) (2024-09-23)
+
+
+### Miscellaneous
+
+* set maxIdleConns for jobsdb handles and emit (*sql.DB).Stats ([#5123](https://github.com/rudderlabs/rudder-server/issues/5123)) ([2aafc50](https://github.com/rudderlabs/rudder-server/commit/2aafc5054295b42b107d94543f50e0cabdb24fdd))
+
 ## [1.34.0](https://github.com/rudderlabs/rudder-server/compare/v1.33.1...v1.34.0) (2024-09-18)
 
 

integration_test/warehouse/testdata/docker-compose.ssh-server.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   ssh-server:
     image: lscr.io/linuxserver/openssh-server:9.3_p2-r1-ls145

jobsdb/jobsdb.go

@@ -43,6 +43,7 @@ import (
 	"github.com/tidwall/gjson"
 
 	"github.com/rudderlabs/rudder-go-kit/bytesize"
+	"github.com/rudderlabs/rudder-go-kit/stats/metric"
 
 	"github.com/rudderlabs/rudder-go-kit/logger"
 	"github.com/rudderlabs/rudder-server/jobsdb/internal/cache"
@@ -743,30 +744,31 @@ func (jd *Handle) init() {
 	// Initialize dbHandle if not already set
 	if jd.dbHandle == nil {
 		var err error
-		psqlInfo := misc.GetConnectionString(jd.config, "jobsdb")
+		psqlInfo := misc.GetConnectionString(jd.config, "jobsdb_"+jd.tablePrefix)
 		jd.dbHandle, err = sql.Open("postgres", psqlInfo)
 		jd.assertError(err)
 	}
 
+	var maxConns int
 	if !jd.conf.enableReaderQueue || !jd.conf.enableWriterQueue {
-		jd.dbHandle.SetMaxOpenConns(jd.conf.maxOpenConnections)
+		maxConns = jd.conf.maxOpenConnections
 	} else {
-		maxOpenConnections := 2 // buffer
-		maxOpenConnections += jd.conf.maxReaders + jd.conf.maxWriters
+		maxConns = 2 // buffer
+		maxConns += jd.conf.maxReaders + jd.conf.maxWriters
 		switch jd.ownerType {
 		case Read:
-			maxOpenConnections += 2 // migrate, refreshDsList
+			maxConns += 2 // migrate, refreshDsList
 		case Write:
-			maxOpenConnections += 1 // addNewDS
+			maxConns += 1 // addNewDS
 		case ReadWrite:
-			maxOpenConnections += 3 // migrate, addNewDS, archive
+			maxConns += 3 // migrate, addNewDS, archive
 		}
-		if maxOpenConnections < jd.conf.maxOpenConnections {
-			jd.dbHandle.SetMaxOpenConns(maxOpenConnections)
-		} else {
-			jd.dbHandle.SetMaxOpenConns(jd.conf.maxOpenConnections)
+		if maxConns >= jd.conf.maxOpenConnections {
+			maxConns = jd.conf.maxOpenConnections
 		}
 	}
+	jd.dbHandle.SetMaxOpenConns(maxConns)
+	jd.dbHandle.SetMaxIdleConns(maxConns)
 
 	jd.assertError(jd.dbHandle.Ping())
 
@@ -969,9 +971,61 @@ func (jd *Handle) Start() error {
 	if !jd.skipSetupDBSetup {
 		jd.setUpForOwnerType(ctx, jd.ownerType)
 	}
+	g.Go(crash.Wrapper(func() error {
+		for {
+			select {
+			case <-time.After(jd.config.GetDuration("JobsDB.ConnStatsFrequency", 15, time.Second)):
+			case <-ctx.Done():
+				return nil
+			}
+			stats := jd.dbHandle.Stats()
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_max_open_connections", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.MaxOpenConnections))
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_open_connections", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.OpenConnections))
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_in_use_connections", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.InUse))
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_idle_connections", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.Idle))
+
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_wait_count_total", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.WaitCount))
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_wait_duration_seconds_total", tablePrefix: jd.tablePrefix},
+			).Set(stats.WaitDuration.Seconds())
+
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_max_idle_closed_total", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.MaxIdleClosed))
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_max_idle_time_closed_total", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.MaxIdleTimeClosed))
+			metric.Instance.GetRegistry(metric.PublishedMetrics).MustGetGauge(
+				dbConnStats{name: "sql_db_max_lifetime_closed_total", tablePrefix: jd.tablePrefix},
+			).Set(float64(stats.MaxLifetimeClosed))
+
+		}
+	}))
 	return nil
 }
 
+type dbConnStats struct {
+	name, tablePrefix string
+}
+
+func (dbs dbConnStats) GetName() string {
+	return dbs.name
+}
+
+func (dbs dbConnStats) GetTags() map[string]string {
+	return map[string]string{"name": "jobsdb_" + dbs.tablePrefix}
+}
+
 func (jd *Handle) setUpForOwnerType(ctx context.Context, ownerType OwnerType) {
 	jd.dsListLock.WithLock(func(l lock.LockToken) {
 		switch ownerType {

warehouse/integrations/azure-synapse/azure_synapse_test.go

@@ -41,6 +41,7 @@ import (
 )
 
 func TestIntegration(t *testing.T) {
+	t.Skip("skipping for 1.34.1 release")
 	if os.Getenv("SLOW") != "1" {
 		t.Skip("Skipping tests. Add 'SLOW=1' env var to run test.")
 	}

warehouse/integrations/azure-synapse/testdata/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   azure_synapse:
     image: rudderstack/azure-sql-edge:1.0.6

warehouse/integrations/mssql/mssql_test.go

@@ -40,6 +40,7 @@ import (
 )
 
 func TestIntegration(t *testing.T) {
+	t.Skip("skipping for 1.34.1 release")
 	if os.Getenv("SLOW") != "1" {
 		t.Skip("Skipping tests. Add 'SLOW=1' env var to run test.")
 	}

warehouse/integrations/mssql/testdata/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   mssql:
     image: rudderstack/azure-sql-edge:1.0.6

warehouse/integrations/postgres/testdata/docker-compose.ssh-server.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   ssh-server:
     image: lscr.io/linuxserver/openssh-server:9.3_p2-r1-ls145

warehouse/integrations/testdata/docker-compose.jobsdb.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   jobsDb:
     image: postgres:latest

warehouse/integrations/testdata/docker-compose.minio.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   minio:
     image: minio/minio:latest

warehouse/integrations/tunnelling/testdata/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   openssh-server:
     image: lscr.io/linuxserver/openssh-server:9.3_p2-r1-ls145