From 928dea83d23495de9231df502924eaf1b21b58e5 Mon Sep 17 00:00:00 2001 From: kangxiang Date: Mon, 23 Sep 2024 11:11:12 +0800 Subject: [PATCH] feat: default genesis sync vpc name --- server/controller/cloud/config/config.go | 1 - server/controller/cloud/genesis/genesis.go | 12 ------------ server/controller/cloud/genesis/network.go | 8 +------- server/controller/cloud/genesis/subnet.go | 8 +------- server/controller/cloud/genesis/vinterface.go | 8 +------- server/controller/cloud/genesis/vm.go | 7 +------ server/controller/cloud/genesis/vpc.go | 7 +------ server/controller/genesis/config/config.go | 1 + server/controller/genesis/updater/sync.go | 12 +++++++----- server/server.yaml | 7 ++++--- 10 files changed, 17 insertions(+), 54 deletions(-) diff --git a/server/controller/cloud/config/config.go b/server/controller/cloud/config/config.go index 85137870a12..1ec3d3c5b89 100644 --- a/server/controller/cloud/config/config.go +++ b/server/controller/cloud/config/config.go @@ -29,7 +29,6 @@ type CloudConfig struct { KubernetesGatherInterval uint32 `default:"30" yaml:"kubernetes_gather_interval"` AliyunRegionName string `default:"cn-beijing" yaml:"aliyun_region_name"` AWSRegionName string `default:"cn-north-1" yaml:"aws_region_name"` - GenesisDefaultVpcName string `default:"default_vpc" yaml:"genesis_default_vpc"` HostnameToIPFile string `default:"/etc/hostname_to_ip.csv" yaml:"hostname_to_ip_file"` DNSEnable bool `default:"false" yaml:"dns_enable"` HTTPTimeout int `default:"30" yaml:"http_timeout"` diff --git a/server/controller/cloud/genesis/genesis.go b/server/controller/cloud/genesis/genesis.go index 0ba5e78e6a5..a2c8a4720a4 100644 --- a/server/controller/cloud/genesis/genesis.go +++ b/server/controller/cloud/genesis/genesis.go @@ -42,13 +42,11 @@ type Genesis struct { teamID int ipV4CIDRMaxMask int ipV6CIDRMaxMask int - defaultVpc bool Name string Lcuuid string UuidGenerate string regionLcuuid string azLcuuid string - defaultVpcName string ips []cloudmodel.IP subnets []cloudmodel.Subnet genesisData gcommon.GenesisSyncDataResponse @@ -81,7 +79,6 @@ func NewGenesis(orgID int, domain mysqlmodel.Domain, cfg config.CloudConfig) (*G Name: domain.Name, Lcuuid: domain.Lcuuid, UuidGenerate: domain.DisplayName, - defaultVpcName: cfg.GenesisDefaultVpcName, regionLcuuid: regionLcuuid, genesisData: gcommon.GenesisSyncDataResponse{}, cloudStatsd: statsd.NewCloudStatsd(), @@ -268,7 +265,6 @@ func (g *Genesis) generateIPsAndSubnets() { func (g *Genesis) GetCloudData() (cloudmodel.Resource, error) { g.azLcuuid = "" - g.defaultVpc = false g.cloudStatsd = statsd.NewCloudStatsd() if genesis.GenesisService == nil { @@ -322,14 +318,6 @@ func (g *Genesis) GetCloudData() (cloudmodel.Resource, error) { if err != nil { return cloudmodel.Resource{}, err } - if g.defaultVpc { - vpc := cloudmodel.VPC{ - Lcuuid: common.GetUUIDByOrgID(g.orgID, g.defaultVpcName), - Name: g.defaultVpcName, - RegionLcuuid: g.regionLcuuid, - } - vpcs = append(vpcs, vpc) - } resource := cloudmodel.Resource{ IPs: ips, diff --git a/server/controller/cloud/genesis/network.go b/server/controller/cloud/genesis/network.go index 65e52c8a14e..2c5c3e1e266 100644 --- a/server/controller/cloud/genesis/network.go +++ b/server/controller/cloud/genesis/network.go @@ -21,7 +21,6 @@ import ( "time" "github.com/deepflowio/deepflow/server/controller/cloud/model" - "github.com/deepflowio/deepflow/server/controller/common" "github.com/deepflowio/deepflow/server/libs/logger" ) @@ -37,11 +36,6 @@ func (g *Genesis) getNetworks() ([]model.Network, error) { log.Debug("segmentation id not found", logger.NewORGPrefix(g.orgID)) continue } - vpcLcuuid := n.VPCLcuuid - if vpcLcuuid == "" { - vpcLcuuid = common.GetUUIDByOrgID(g.orgID, g.defaultVpcName) - g.defaultVpc = true - } networkName := n.Name if networkName == "" { networkName = "subnet_vni_" + strconv.Itoa(int(n.SegmentationID)) @@ -50,7 +44,7 @@ func (g *Genesis) getNetworks() ([]model.Network, error) { Lcuuid: n.Lcuuid, Name: networkName, SegmentationID: int(n.SegmentationID), - VPCLcuuid: vpcLcuuid, + VPCLcuuid: n.VPCLcuuid, Shared: false, External: n.External, NetType: int(n.NetType), diff --git a/server/controller/cloud/genesis/subnet.go b/server/controller/cloud/genesis/subnet.go index 5ef6088c941..b52e4cb6073 100644 --- a/server/controller/cloud/genesis/subnet.go +++ b/server/controller/cloud/genesis/subnet.go @@ -20,7 +20,6 @@ import ( "time" "github.com/deepflowio/deepflow/server/controller/cloud/model" - "github.com/deepflowio/deepflow/server/controller/common" "github.com/deepflowio/deepflow/server/libs/logger" ) @@ -35,11 +34,6 @@ func (g *Genesis) getSubnets() ([]model.Subnet, error) { log.Debug("network lcuuid not found", logger.NewORGPrefix(g.orgID)) continue } - vpcLcuuid := s.VPCLcuuid - if vpcLcuuid == "" { - vpcLcuuid = common.GetUUIDByOrgID(g.orgID, g.defaultVpcName) - g.defaultVpc = true - } subnetName := s.Name if subnetName == "" { subnetName = "subnet_" + s.Lcuuid[:11] @@ -48,7 +42,7 @@ func (g *Genesis) getSubnets() ([]model.Subnet, error) { Lcuuid: s.Lcuuid, Name: subnetName, CIDR: s.CIDR, - VPCLcuuid: vpcLcuuid, + VPCLcuuid: s.VPCLcuuid, NetworkLcuuid: s.NetworkLcuuid, } subnets = append(subnets, subnet) diff --git a/server/controller/cloud/genesis/vinterface.go b/server/controller/cloud/genesis/vinterface.go index 90e109039e4..b143f23d32b 100644 --- a/server/controller/cloud/genesis/vinterface.go +++ b/server/controller/cloud/genesis/vinterface.go @@ -20,7 +20,6 @@ import ( "time" "github.com/deepflowio/deepflow/server/controller/cloud/model" - "github.com/deepflowio/deepflow/server/controller/common" "github.com/deepflowio/deepflow/server/libs/logger" ) @@ -36,16 +35,11 @@ func (g *Genesis) getVinterfaces() ([]model.VInterface, error) { log.Debug("device lcuuid or network lcuuid not found", logger.NewORGPrefix(g.orgID)) continue } - vpcLcuuid := v.VPCLcuuid - if vpcLcuuid == "" { - vpcLcuuid = common.GetUUIDByOrgID(g.orgID, g.defaultVpcName) - g.defaultVpc = true - } vinterface := model.VInterface{ Lcuuid: v.Lcuuid, Type: int(v.Type), Mac: v.Mac, - VPCLcuuid: vpcLcuuid, + VPCLcuuid: v.VPCLcuuid, RegionLcuuid: g.regionLcuuid, DeviceType: int(v.DeviceType), DeviceLcuuid: v.DeviceLcuuid, diff --git a/server/controller/cloud/genesis/vm.go b/server/controller/cloud/genesis/vm.go index 4d0c910db57..36dcfe786d6 100644 --- a/server/controller/cloud/genesis/vm.go +++ b/server/controller/cloud/genesis/vm.go @@ -32,11 +32,6 @@ func (g *Genesis) getVMs() ([]model.VM, error) { g.cloudStatsd.RefreshAPIMoniter("vms", len(vmsData), time.Time{}) for _, v := range vmsData { - vpcLcuuid := v.VPCLcuuid - if vpcLcuuid == "" { - vpcLcuuid = common.GetUUIDByOrgID(g.orgID, g.defaultVpcName) - g.defaultVpc = true - } launchServer := v.LaunchServer if launchServer == "127.0.0.1" { launchServer = "" @@ -46,7 +41,7 @@ func (g *Genesis) getVMs() ([]model.VM, error) { Name: v.Name, Label: v.Label, HType: common.VM_HTYPE_VM_C, - VPCLcuuid: vpcLcuuid, + VPCLcuuid: v.VPCLcuuid, State: int(v.State), LaunchServer: launchServer, CreatedAt: v.CreatedAt, diff --git a/server/controller/cloud/genesis/vpc.go b/server/controller/cloud/genesis/vpc.go index eba9ddad3cb..fb7373339f3 100644 --- a/server/controller/cloud/genesis/vpc.go +++ b/server/controller/cloud/genesis/vpc.go @@ -20,7 +20,6 @@ import ( "time" "github.com/deepflowio/deepflow/server/controller/cloud/model" - "github.com/deepflowio/deepflow/server/controller/common" "github.com/deepflowio/deepflow/server/libs/logger" ) @@ -32,12 +31,8 @@ func (g *Genesis) getVPCs() ([]model.VPC, error) { g.cloudStatsd.RefreshAPIMoniter("vpcs", len(vpcsData), time.Time{}) for _, v := range vpcsData { - vpcLcuuid := v.Lcuuid - if vpcLcuuid == "" { - vpcLcuuid = common.GetUUIDByOrgID(g.orgID, v.Name) - } vpc := model.VPC{ - Lcuuid: vpcLcuuid, + Lcuuid: v.Lcuuid, Name: v.Name, RegionLcuuid: g.regionLcuuid, } diff --git a/server/controller/genesis/config/config.go b/server/controller/genesis/config/config.go index 2ea55d153b2..e20cbb50efe 100644 --- a/server/controller/genesis/config/config.go +++ b/server/controller/genesis/config/config.go @@ -26,6 +26,7 @@ type GenesisConfig struct { DataPersistenceInterval int `default:"60" yaml:"data_persistence_interval"` MultiNSMode bool `default:"false" yaml:"multi_ns_mode"` SingleVPCMode bool `default:"false" yaml:"single_vpc_mode"` + DefaultVPCName string `default:"default-public-vpc" yaml:"default_vpc_name"` IgnoreNICRegex string `default:"^(kube-ipvs)" yaml:"ignore_nic_regex"` VMNameField string `default:"metadata" yaml:"vm_name_field"` } diff --git a/server/controller/genesis/updater/sync.go b/server/controller/genesis/updater/sync.go index 911096b89cb..adfc7d4161d 100644 --- a/server/controller/genesis/updater/sync.go +++ b/server/controller/genesis/updater/sync.go @@ -47,6 +47,7 @@ type GenesisSyncRpcUpdater struct { multiNSMode bool singleVPCMode bool nodeIP string + defaultVPCName string vmNameField string ignoreNICRegex *regexp.Regexp } @@ -130,6 +131,7 @@ func NewGenesisSyncRpcUpdater(cfg config.GenesisConfig) *GenesisSyncRpcUpdater { multiNSMode: cfg.MultiNSMode, singleVPCMode: cfg.SingleVPCMode, vmNameField: cfg.VMNameField, + defaultVPCName: cfg.DefaultVPCName, ignoreNICRegex: ignoreNICRegex, } } @@ -368,15 +370,15 @@ func (v *GenesisSyncRpcUpdater) ParseHostAsVmPlatformInfo(info common.VIFRPCMess log.Error(err.Error(), logger.NewORGPrefix(info.ORGID)) return common.GenesisSyncDataResponse{} } - // check if vm is behind NAT - behindNat := peer != natIP - log.Infof("host (%s) nat ip is (%s) peer ip is (%s), behind nat: (%t), single vpc mode: (%t)", hostName, natIP, peer, behindNat, v.singleVPCMode, logger.NewORGPrefix(info.ORGID)) vpc := model.GenesisVpc{ - Name: "default-public-cloud-vpc", - Lcuuid: ccommon.GetUUIDByOrgID(info.ORGID, "default-public-cloud-vpc"), + Name: v.defaultVPCName, + Lcuuid: ccommon.GetUUIDByOrgID(info.ORGID, v.defaultVPCName), VtapID: vtapID, NodeIP: v.nodeIP, } + // check if vm is behind NAT + behindNat := peer != natIP + log.Infof("host (%s) nat ip is (%s) peer ip is (%s), behind nat: (%t), single vpc mode: (%t)", hostName, natIP, peer, behindNat, v.singleVPCMode, logger.NewORGPrefix(info.ORGID)) if behindNat && !v.singleVPCMode { vpc = model.GenesisVpc{ Name: "VPC-" + peer, diff --git a/server/server.yaml b/server/server.yaml index 9d42a2392c2..626eadafac2 100644 --- a/server/server.yaml +++ b/server/server.yaml @@ -203,8 +203,6 @@ controller: aliyun_region_name: cn-beijing # AWS API获取区域列表时，需要指定一个区域，并通过这个区域区分国际版和国内版 aws_region_name: cn-north-1 - # 采集器同步默认vpc名称 - genesis_default_vpc: default_vpc # 配置宿主机IP文件 hostname_to_ip_file: /etc/hostname_to_ip.csv # 开启debug模式，支持debug云平台API/SDK原始数据 @@ -296,6 +294,9 @@ controller: ipv4_cidr_max_mask: 24 ipv6_cidr_max_mask: 64 + # 采集器同步默认vpc名称 + default_vpc_name: default-public-vpc + # 采集器上报数据处理的队列长度 queue_length: 1000 @@ -326,7 +327,7 @@ controller: ignore_nic_regex: ## 采集器同步 KVM ，解析虚拟机的 XML 时，默认为 metadata 即使用 domian.metadata.instance.name 字段作为虚拟机的名称 - ## 可通过此配置将虚拟机名称指定为 domain.name、domain.title、domain.uuid 之一，默认为空 + ## 可通过此配置将虚拟机名称指定为 domain.name、domain.title、domain.uuid 之一 ## 当使用此配置指定的字段取不到虚拟机名称或虚拟机名称为空时，则将使用 domain.name 的值作为虚拟机名称 ## 例：vm_name_field: title ，指定 domain.title 作为虚拟机的名称 #vm_name_field: metadata