Merge branch 'main' into remove-old-official-images

.github/workflows/ci.yml

@@ -238,7 +238,7 @@ jobs:
           name: official-oss-integration-test-logs-${{ matrix.container.image }}-${{ matrix.container.version }}
           path: /tmp/integration-test-logs/
           retention-days: 3
-          
+
   official-plus-image-integration-tests:
       name: Integration Tests - Official Plus Images
       needs: build-unsigned-snapshot

.github/workflows/f5-cla.yml

@@ -0,0 +1,51 @@
+name: F5 CLA
+
+on:
+  issue_comment:
+    types:
+      - created
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+concurrency:
+  group: ${{ github.ref_name }}-cla
+
+permissions:
+  contents: read
+
+jobs:
+  f5-cla:
+    name: F5 CLA
+    runs-on: ubuntu-22.04
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: write
+      statuses: write
+    steps:
+      - name: Run F5 Contributor License Agreement (CLA) assistant
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have hereby read the F5 CLA and agree to its terms') || github.event_name == 'pull_request_target'
+        uses: contributor-assistant/github-action@f41946747f85d28e9a738f4f38dbcc74b69c7e0e # v2.5.1
+        with:
+          # Any pull request targeting the following branch will trigger a CLA check.
+          branch: "main"
+          # Path to the CLA document.
+          path-to-document: "https://github.com/f5/.github/blob/main/CLA/cla-markdown.md"
+          # Custom CLA messages.
+          custom-notsigned-prcomment: "🎉 Thank you for your contribution! It appears you have not yet signed the F5 Contributor License Agreement (CLA), which is required for your changes to be incorporated into an F5 Open Source Software (OSS) project. Please kindly read the [F5 CLA](https://github.com/f5/.github/blob/main/CLA/cla-markdown.md) and reply on a new comment with the following text to agree:"
+          custom-pr-sign-comment: "I have hereby read the F5 CLA and agree to its terms"
+          custom-allsigned-prcomment: "✅ All required contributors have signed the F5 CLA for this PR. Thank you!"
+          # Remote repository storing CLA signatures.
+          remote-organization-name: "f5"
+          remote-repository-name: "f5-cla-data"
+          path-to-signatures: "signatures/beta/signatures.json"
+          # Comma separated list of usernames for maintainers or any other individuals who should not be prompted for a CLA.
+          allowlist: bot*
+          # Do not lock PRs after a merge.
+          lock-pullrequest-aftermerge: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.F5_CLA_TOKEN }}

.github/workflows/release-branch.yml

@@ -10,11 +10,6 @@ on:
         required: true
         type: boolean
         default: false
-      createPullRequest:
-        description: 'Create pull request back into main'
-        required: true
-        type: boolean
-        default: false
       uploadJWT:
         description: 'Temporary JWT to publish packages to up-ap.nginx.com'
         required: true
@@ -125,12 +120,27 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y gpgv1 monkeysphere 
-          go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
-      - name: Tag release
+          go install github.com/goreleaser/nfpm/v2/cmd/[email protected]
+      - name: Generate Changelog
+        env:
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
+          git clone https://github.com/nginx/agent-changelog.git
+          cd ./agent-changelog/source
+          pip install -r requirements.txt
+          python agent.py
+      - name: Push Changelog
+        run: |
+          mv agent-changelog/source/changelog.md ./site/content/
+          
           git config --global user.name 'github-actions'
           git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
 
+          git add ./site/content/changelog.md
+          git commit -m "Add generated changelog"
+          git push origin HEAD:${{ github.ref_name }}
+      - name: Tag release
+        run: |
           git tag -a "v${{env.VERSION}}" -m "CI Autogenerated"
           git tag -a "sdk/v${{env.VERSION}}" -m "CI Autogenerated"
       - name: Push Tags
@@ -175,8 +185,6 @@ jobs:
           az logout
         if: always()
       - name: Upload Release Assets
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         # clobber overwrites existing assets of the same name
         run: |
           gh release upload --clobber v${{env.VERSION}} \
@@ -202,7 +210,7 @@ jobs:
             }))
             console.log(`Release published: ${release.data.html_url}`)
       - name: Create Pull Request
-        if: ${{ inputs.publishPackages == true && inputs.createPullRequest == true }}
+        if: ${{ inputs.publishPackages == true }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           script: |

CONTRIBUTING.md

@@ -35,6 +35,15 @@ To suggest a feature or enhancement, please create an issue on GitHub with the l
 
 Note: if you'd like to implement a new feature, please consider creating a feature request issue first to start a discussion about the feature.
 
+### F5 Contributor License Agreement (CLA)
+
+F5 requires all external contributors to agree to the terms of the F5 CLA (available [here](https://github.com/f5/.github/blob/main/CLA/cla-markdown.md))
+before any of their changes can be incorporated into an F5 Open Source repository.
+
+If you have not yet agreed to the F5 CLA terms and submit a PR to this repository, a bot will prompt you to view and
+agree to the F5 CLA. You will have to agree to the F5 CLA terms through a comment in the PR before any of your changes
+can be merged. Your agreement signature will be safely stored by F5 and no longer be required in future PRs.
+
 ## Code Guidelines
 
 <!-- ### Go Guidelines Here Linter-->

Makefile.packaging

@@ -182,11 +182,18 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
 gpg-key: ## Generate GPG public key
 	$$(gpg --import $(NFPM_SIGNING_KEY_FILE)); \
 	keyid=$$(gpg --list-keys NGINX | egrep -A1 "^pub" | egrep -v "^pub" | tr -d '[:space:]'); \
+	if [ -z "$$keyid" ]; then echo "Error: GPG key not found."; exit 1; fi; \
+	# Check if the key is expired \
+	# Look for the 'e' (expired) flag in the 'pub' or 'uid' lines \
+	if gpg --list-keys --with-colons "$$keyid" | grep -E '^pub:e:|^uid:e:'; then \
+		echo "Error: GPG key has expired."; \
+		exit 1; \
+	fi; \
 	expiry=1y; \
 	$$(gpg --quick-set-expire $$keyid $$expiry '*'); \
-	# we need to convert the private gpg key to rsa pem format for pkg signing \
+	# Convert the private GPG key to RSA PEM format for pkg signing \
 	$$(gpg --export-secret-key $$keyid | openpgp2ssh $$keyid > .key.rsa); \
-	$$(gpg --output $(GPG_PUBLIC_KEY) --armor --export)
+	$$(gpg --output $(GPG_PUBLIC_KEY) --armor --export $$keyid)
 
 release: ## Publish tarball to the UPLOAD_URL
 	echo "Publishing nginx-agent packages to ${UPLOAD_URL}"; \

nginx-agent.conf

@@ -46,4 +46,4 @@ config_dirs: "/etc/nginx:/usr/local/etc/nginx:/usr/share/nginx/modules:/etc/nms"
   # host: 127.0.0.1
   # 
   # Set this value to a secure port number to prevent information leaks. 
-  # port: 8038
+  # port: 8038

sdk/client/metric_reporter.go

@@ -122,6 +122,8 @@ func (r *metricReporter) createClient() error {
 }
 
 func (r *metricReporter) Close() (err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
 	return r.closeConnection()
 }
 

sdk/config_helpers.go

@@ -24,6 +24,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/nginx/agent/sdk/v2/backoff"
@@ -43,6 +44,8 @@ const (
 	httpClientTimeout         = 1 * time.Second
 )
 
+var readLock = sync.Mutex{}
+
 type DirectoryMap struct {
 	paths map[string]*proto.Directory
 }
@@ -113,6 +116,7 @@ func GetNginxConfigWithIgnoreDirectives(
 	allowedDirectories map[string]struct{},
 	ignoreDirectives []string,
 ) (*proto.NginxConfig, error) {
+	readLock.Lock()
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
 			IgnoreDirectives:   ignoreDirectives,
@@ -142,6 +146,7 @@ func GetNginxConfigWithIgnoreDirectives(
 	if err != nil {
 		return nil, fmt.Errorf("error assemble payload from %s, error: %s", confFile, err)
 	}
+	readLock.Unlock()
 
 	return nginxConfig, nil
 }

src/core/config/config.go

@@ -19,6 +19,7 @@ import (
 	"reflect"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	agent_config "github.com/nginx/agent/sdk/v2/agent/config"
@@ -50,6 +51,7 @@ const (
 var (
 	Viper       = viper.NewWithOptions(viper.KeyDelimiter(agent_config.KeyDelimiter))
 	MigratedEnv = false
+	cfgMu       = sync.Mutex{}
 )
 
 func SetVersion(version, commit string) {
@@ -196,7 +198,7 @@ func RegisterRunner(r func(cmd *cobra.Command, args []string)) {
 }
 
 func GetConfig(clientId string) (*Config, error) {
-	extensions := []string{}
+	var extensions []string
 
 	for _, extension := range Viper.GetStringSlice(agent_config.ExtensionsKey) {
 		if agent_config.IsKnownExtension(extension) {
@@ -247,6 +249,8 @@ func GetConfig(clientId string) (*Config, error) {
 // overwritten or not.
 func UpdateAgentConfig(systemId string, updateTags []string, updateFeatures []string) (bool, error) {
 	// Get current config on disk
+	cfgMu.Lock()
+	defer cfgMu.Unlock()
 	config, err := GetConfig(systemId)
 	if err != nil {
 		log.Errorf("Failed to register config: %v", err)

src/core/config/config_test.go

@@ -163,7 +163,7 @@ func TestGetConfig(t *testing.T) {
 
 		assert.Equal(t, []string{}, config.Tags)
 		assert.Equal(t, Defaults.Features, config.Features)
-		assert.Equal(t, []string{}, config.Extensions)
+		assert.Equal(t, []string(nil), config.Extensions)
 	})
 
 	t.Run("test override defaults with flags", func(t *testing.T) {
@@ -274,7 +274,7 @@ func TestGetConfig(t *testing.T) {
 		assert.Equal(t, Defaults.AgentMetrics.Mode, config.AgentMetrics.Mode)
 		assert.Equal(t, 10*time.Minute, config.AgentMetrics.Backoff.MaxInterval)
 		assert.Equal(t, Defaults.Features, config.Features)
-		assert.Equal(t, []string{}, config.Extensions)
+		assert.Equal(t, []string(nil), config.Extensions)
 	})
 
 	t.Run("test override config values with ENV variables", func(t *testing.T) {

src/core/environment.go

@@ -20,6 +20,7 @@ import (
 	"regexp"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 
 	"github.com/google/uuid"
@@ -43,7 +44,6 @@ import (
 //go:generate mv fake_environment_fixed.go fake_environment_test.go
 type Environment interface {
 	NewHostInfo(agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo
-	// NewHostInfoWithContext(agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo
 	GetHostname() (hostname string)
 	GetSystemUUID() (hostId string)
 	ReadDirectory(dir string, ext string) ([]string, error)
@@ -70,6 +70,7 @@ type EnvironmentType struct {
 	host               *proto.HostInfo
 	virtualizationFunc func(ctx context.Context) (string, string, error)
 	isContainerFunc    func() bool
+	hostMu             sync.Mutex
 }
 
 type Process struct {
@@ -106,6 +107,7 @@ const (
 	IsContainerKey      = "isContainer"
 	GetContainerIDKey   = "GetContainerID"
 	GetSystemUUIDKey    = "GetSystemUUIDKey"
+	ReleaseInfoFile     = "/etc/os-release"
 )
 
 var (
@@ -125,6 +127,8 @@ func (env *EnvironmentType) NewHostInfo(agentVersion string, tags *[]string, con
 
 func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo {
 	defer ctx.Done()
+	env.hostMu.Lock()
+	defer env.hostMu.Unlock()
 	// temp cache measure
 	if env.host == nil || clearCache {
 		hostInformation, err := host.InfoWithContext(ctx)
@@ -154,7 +158,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer
 			Partitons:           disks,
 			Network:             env.networks(),
 			Processor:           env.processors(hostInformation.KernelArch),
-			Release:             releaseInfo("/etc/os-release"),
+			Release:             releaseInfo(ReleaseInfoFile),
 			Tags:                *tags,
 			AgentAccessibleDirs: configDirs,
 		}

src/core/metrics/sources/disk.go

@@ -14,7 +14,6 @@ import (
 	"github.com/nginx/agent/sdk/v2/proto"
 	"github.com/nginx/agent/v2/src/core"
 	"github.com/nginx/agent/v2/src/core/metrics"
-	log "github.com/sirupsen/logrus"
 )
 
 const MOUNT_POINT = "mount_point"
@@ -49,8 +48,6 @@ func (c *Disk) Collect(ctx context.Context, m chan<- *metrics.StatsEntityWrapper
 			"in_use": float64(usage.UsedPercentage),
 		})
 
-		log.Debugf("disk metrics collected: %v", len(simpleMetrics))
-
 		select {
 		case <-ctx.Done():
 			return

src/core/metrics/sources/net_io.go

@@ -12,11 +12,12 @@ import (
 	"fmt"
 	"sync"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/nginx/agent/sdk/v2/proto"
 	"github.com/nginx/agent/v2/src/core"
 	"github.com/nginx/agent/v2/src/core/metrics"
 	"github.com/shirou/gopsutil/v3/net"
-	log "github.com/sirupsen/logrus"
 )
 
 const NETWORK_INTERFACE = "network_interface"
@@ -82,7 +83,6 @@ func (nio *NetIO) Collect(ctx context.Context, m chan<- *metrics.StatsEntityWrap
 		}
 
 		simpleMetrics := nio.convertSamplesToSimpleMetrics(v)
-		log.Debugf("net IO stats count: %d", len(simpleMetrics))
 
 		select {
 		case <-ctx.Done():
@@ -109,6 +109,7 @@ func (nio *NetIO) Collect(ctx context.Context, m chan<- *metrics.StatsEntityWrap
 	simpleMetrics := nio.convertSamplesToSimpleMetrics(totalStats)
 	m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM)
 
+	log.Debugf("net IO stats: %v", currentNetIOStats)
 	nio.netIOStats = currentNetIOStats
 }
 

src/core/metrics/sources/nginx_access_log.go

@@ -182,7 +182,6 @@ func (c *NginxAccessLog) Stop() {
 		fn()
 		delete(c.logs, f)
 	}
-	log.Debugf("Stopping NginxAccessLog source for nginx id: %v", c.baseDimensions.NginxId)
 }
 
 func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *metrics.StatsEntityWrapper) {
@@ -329,11 +328,11 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
 			mu.Unlock()
 
 		case <-tick.C:
+			mu.Lock()
+
 			c.baseDimensions.NginxType = c.nginxType
 			c.baseDimensions.PublishedAPI = logFile
 
-			mu.Lock()
-
 			if len(requestLengths) > 0 {
 				httpCounters["request.length"] = getAverageMetricValue(requestLengths)
 			}