chore: Improve rebuild speed by eliminating unnecessary rerun of buil…

…d.rs - Watch libtrace changes by separate files instead of folders - Unwatch files generated from protobuf sources
deepflowio · Sep 24, 2024 · 7f42010 · 7f42010
1 parent 7be75b3
commit 7f42010
Show file tree

Hide file tree

Showing 7 changed files with 89 additions and 26 deletions.
diff --git a/agent/Cargo.lock b/agent/Cargo.lock
diff --git a/agent/Cargo.toml b/agent/Cargo.toml
@@ -142,9 +142,11 @@ tempfile = "3.2.0"
 uluru = "3.0.0"
 
 [build-dependencies]
+anyhow = "1.0"
 chrono = "0.4"
 dunce = "1"
 tonic-build = "0.8.0"
+walkdir = "2"
 
 [features]
 enterprise = ["extended_profile", "enterprise-utils"]

diff --git a/agent/build.rs b/agent/build.rs
@@ -14,11 +14,18 @@
  * limitations under the License.
  */
 
-use std::{env, error::Error, path::PathBuf, process::Command, str};
+use std::{
+    env,
+    path::{Path, PathBuf},
+    process::Command,
+    str,
+};
 
+use anyhow::Result;
 use chrono::prelude::*;
+use walkdir::WalkDir;
 
-fn get_branch() -> Result<String, Box<dyn Error>> {
+fn get_branch() -> Result<String> {
     if let Ok(branch) = env::var("GITHUB_REF_NAME") {
         return Ok(branch);
     }
@@ -60,7 +67,7 @@ fn get_branch() -> Result<String, Box<dyn Error>> {
 
 struct EnvCommand(&'static str, Vec<&'static str>);
 
-fn set_build_info() -> Result<(), Box<dyn Error>> {
+fn set_build_info() -> Result<()> {
     println!("cargo:rustc-env=AGENT_NAME=deepflow-agent-ce");
     println!("cargo:rustc-env=BRANCH={}", get_branch()?);
     println!(
@@ -79,7 +86,61 @@ fn set_build_info() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
-fn set_build_libtrace() -> Result<(), Box<dyn Error>> {
+// libtrace scatters generated files in different folders, making it difficult to watch a single folder for changes
+//
+// rerun build script when one of the following file changes
+// - C source files, except for
+//   - generated bpf bytecode files (socket_trace_*.c / perf_profiler_*.c)
+//   - java agent so files and jattach bin
+// - Header files
+// - `src/ebpf/mod.rs` (to exlude rust sources in `samples` folder)
+// - Makefiles
+fn set_libtrace_rerun_files() -> Result<()> {
+    fn watched(path: &Path) -> bool {
+        if let Some(ext) = path.extension().and_then(|s| s.to_str()) {
+            match ext {
+                "c" => {
+                    if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
+                        if name.starts_with("socket_trace_") || name.starts_with("perf_profiler_") {
+                            return false;
+                        }
+                        if name.starts_with("java_agent_so_") {
+                            return false;
+                        }
+                        if name == "deepflow_jattach_bin.c" {
+                            return false;
+                        }
+                        return true;
+                    }
+                }
+                "h" => return true,
+                _ => (),
+            }
+        }
+        if path == Path::new("src/ebpf/mods.rs") {
+            return true;
+        }
+        if let Some(name) = path.file_name() {
+            if name == "Makefile" {
+                return true;
+            }
+        }
+        false
+    }
+    let base_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
+    for entry in WalkDir::new(base_dir.join("src/ebpf")) {
+        let entry = entry?;
+        let relative_path = entry.path().strip_prefix(&base_dir)?;
+        if !watched(relative_path) {
+            continue;
+        }
+        println!("cargo:rerun-if-changed={}", relative_path.display());
+    }
+    Ok(())
+}
+
+fn set_build_libtrace() -> Result<()> {
+    set_libtrace_rerun_files()?;
     let output = match env::var("CARGO_CFG_TARGET_ENV")?.as_str() {
         "gnu" => Command::new("sh").arg("-c")
             .arg("cd src/ebpf && make clean && make --no-print-directory && make tools --no-print-directory")
@@ -104,7 +165,7 @@ fn set_build_libtrace() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
-fn set_linkage() -> Result<(), Box<dyn Error>> {
+fn set_linkage() -> Result<()> {
     let target_env = env::var("CARGO_CFG_TARGET_ENV")?;
     if target_env.as_str() == "musl" {
         #[cfg(target_arch = "x86_64")]
@@ -155,24 +216,28 @@ fn set_linkage() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
-fn compile_wasm_plugin_proto() -> Result<(), Box<dyn Error>> {
+fn compile_wasm_plugin_proto() -> Result<()> {
     tonic_build::configure()
         .build_server(false)
+        .emit_rerun_if_changed(false)
         .out_dir("src/plugin/wasm")
-        .compile(&["./src/plugin/WasmPluginApi.proto"], &["./src/plugin"])?;
+        .compile(&["src/plugin/WasmPluginApi.proto"], &["src/plugin"])?;
+    println!("cargo:rerun-if-changed=src/plugin/WasmPluginApi.proto");
     Ok(())
 }
 
-fn make_pulsar_proto() -> Result<(), Box<dyn Error>> {
+fn make_pulsar_proto() -> Result<()> {
     tonic_build::configure()
         .field_attribute(".", "#[serde(skip_serializing_if = \"Option::is_none\")]")
         .type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]")
         .build_server(false)
+        .emit_rerun_if_changed(false)
         .out_dir("src/flow_generator/protocol_logs/mq")
         .compile(
             &["src/flow_generator/protocol_logs/mq/PulsarApi.proto"],
             &["src/flow_generator/protocol_logs/mq"],
         )?;
+    println!("cargo:rerun-if-changed=src/flow_generator/protocol_logs/mq/PulsarApi.proto");
 
     // remove `#[serde(skip_serializing_if = "Option::is_none")]` for non-optional fields
     let filename = "src/flow_generator/protocol_logs/mq/pulsar.proto.rs";
@@ -187,37 +252,26 @@ fn make_pulsar_proto() -> Result<(), Box<dyn Error>> {
         new_lines.push(a[1]);
     }
     std::fs::write(filename, new_lines.join("\n"))?;
-    Command::new("cargo")
-        .args([
-            "fmt",
-            "--",
-            "src/flow_generator/protocol_logs/mq/pulsar.proto.rs",
-        ])
-        .spawn()?;
     Ok(())
 }
 
-fn make_brpc_proto() -> Result<(), Box<dyn Error>> {
+fn make_brpc_proto() -> Result<()> {
     tonic_build::configure()
         .type_attribute(".", "#[derive(serde::Serialize,serde::Deserialize)]")
         .build_server(false)
+        .emit_rerun_if_changed(false)
         .out_dir("src/flow_generator/protocol_logs/rpc/brpc")
         .compile(
             &["src/flow_generator/protocol_logs/rpc/brpc/baidu_rpc_meta.proto"],
             &["src/flow_generator/protocol_logs/rpc"],
         )?;
-
-    Command::new("cargo")
-        .args([
-            "fmt",
-            "--",
-            "src/flow_generator/protocol_logs/rpc/brpc.proto.rs",
-        ])
-        .spawn()?;
+    println!(
+        "cargo:rerun-if-changed=src/flow_generator/protocol_logs/rpc/brpc/baidu_rpc_meta.proto"
+    );
     Ok(())
 }
 
-fn main() -> Result<(), Box<dyn Error>> {
+fn main() -> Result<()> {
     set_build_info()?;
     compile_wasm_plugin_proto()?;
     make_pulsar_proto()?;

diff --git a/agent/src/flow_generator/protocol_logs/mq/pulsar.rs b/agent/src/flow_generator/protocol_logs/mq/pulsar.rs
@@ -1,4 +1,5 @@
 #[path = "pulsar.proto.rs"]
+#[rustfmt::skip]
 mod pulsar_proto;
 
 use prost::Message;

diff --git a/agent/src/flow_generator/protocol_logs/rpc/brpc.rs b/agent/src/flow_generator/protocol_logs/rpc/brpc.rs
@@ -1,4 +1,5 @@
 #[path = "brpc/brpc.policy.rs"]
+#[rustfmt::skip]
 mod brpc_policy;
 
 use brpc_policy::RpcMeta;

diff --git a/agent/src/platform/platform_synchronizer/linux.rs b/agent/src/platform/platform_synchronizer/linux.rs
@@ -35,7 +35,10 @@ use crate::{
     utils::lru::Lru,
 };
 use public::{
-    proto::{agent, trident::{GpidSyncRequest, GpidSyncResponse}},
+    proto::{
+        agent,
+        trident::{GpidSyncRequest, GpidSyncResponse},
+    },
     queue::Receiver,
 };
 

diff --git a/agent/src/plugin/wasm/mod.rs b/agent/src/plugin/wasm/mod.rs
@@ -106,6 +106,7 @@
 */
 
 #[path = "wasm_plugin.rs"]
+#[rustfmt::skip]
 pub mod wasm_plugin;
 
 mod abi_export;