Merge branch 'main' into eliminate_union

.github/workflows/release.yml

@@ -130,15 +130,15 @@ jobs:
           cp ./target/${{ matrix.target }}/${{ env.BUILD_PROFILE }}/databend-* ./target/${{ env.BUILD_PROFILE }}/
           bash ./scripts/ci/ci-run-sqllogic-tests.sh base
 
-  build_udf:
-    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws]
+  build_musl:
+    runs-on: [self-hosted, X64, Linux, 16c32g, aws]
     needs: create_release
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - { target: x86_64-unknown-linux-gnu, runner: X64 }
-          - { target: aarch64-unknown-linux-gnu, runner: ARM64 }
+        target:
+          - x86_64-unknown-linux-musl
+          - aarch64-unknown-linux-musl
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -152,11 +152,9 @@ jobs:
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
-          artifacts: sqllogictests,sqlsmith,metactl,meta,query
-          features: python-udf
-          category: udf
+          artifacts: query,meta,metactl
 
-  build_hdfs:
+  build_udf:
     runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws]
     needs: create_release
     strategy:
@@ -179,28 +177,24 @@ jobs:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
           artifacts: sqllogictests,sqlsmith,metactl,meta,query
-          features: storage-hdfs
-          category: hdfs
+          features: python-udf
+          category: udf
 
   publish:
-    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c8g, aws]
-    needs: [create_release, build_default, build_hdfs]
+    runs-on: [self-hosted, X64, Linux, 4c8g, aws]
+    needs: [create_release, build_default, build_musl]
     strategy:
       fail-fast: false
       matrix:
         include:
           - category: default
             target: x86_64-unknown-linux-gnu
-            runner: X64
           - category: default
             target: aarch64-unknown-linux-gnu
-            runner: ARM64
-          - category: hdfs
-            target: x86_64-unknown-linux-gnu
-            runner: X64
-          - category: hdfs
-            target: aarch64-unknown-linux-gnu
-            runner: ARM64
+          - category: default
+            target: x86_64-unknown-linux-musl
+          - category: default
+            target: aarch64-unknown-linux-musl
     steps:
       - name: Checkout
         uses: actions/checkout@v4

Cargo.toml

@@ -310,7 +310,7 @@ serde_with = { version = "3.8.1" }
 serfig = "0.1.0"
 sled = { version = "0.34", default-features = false }
 stream-more = "0.1.3"
-tantivy = "0.22.0"
+tantivy = { git = "https://github.com/b41sh/tantivy", rev = "37aeac0" }
 thiserror = { version = "1" }
 tikv-jemalloc-ctl = { version = "0.5.0", features = ["use_std"] }
 tokio = { version = "1.35.0", features = ["full"] }

scripts/ci/deploy/config/databend-query-node-otlp-logs.toml

@@ -52,14 +52,14 @@ dir = "./.databend/logs_1"
 
 [log.query]
 on = true
-otlp_endpoint = "http://127.0.0.1:4317/v1/logs"
+otlp_endpoint = "http://127.0.0.1:4317"
 [log.query.otlp_labels]
 qkey1 = "qvalue1"
 qkey2 = "qvalue2"
 
 [log.profile]
 on = true
-otlp_endpoint = "http://127.0.0.1:4318/v1/logs"
+otlp_endpoint = "http://127.0.0.1:4318"
 otlp_protocol = "http"
 [log.profile.otlp_labels]
 pkey1 = "pvalue1"

src/common/io/src/constants.rs

@@ -32,6 +32,8 @@ pub const DEFAULT_BLOCK_INDEX_BUFFER_SIZE: usize = 300 * 1024;
 pub const DEFAULT_BLOCK_MAX_ROWS: usize = 1000 * 1000;
 // The min number of a block by default.
 pub const DEFAULT_BLOCK_MIN_ROWS: usize = 800 * 1000;
+/// The number of bytes read at the end of the file on first read
+pub const DEFAULT_FOOTER_READ_SIZE: u64 = 64 * 1024;
 
 // The min values of table option data_retention_period_in_hours
 pub const DEFAULT_MIN_TABLE_LEVEL_DATA_RETENTION_PERIOD_IN_HOURS: u64 = 1;

src/common/tracing/Cargo.toml

@@ -11,6 +11,7 @@ doctest = false
 test = true
 
 [dependencies]
+anyhow = { workspace = true }
 backtrace = { workspace = true }
 chrono = { workspace = true }
 color-backtrace = { version = "0.6" }
@@ -22,7 +23,7 @@ fastrace-opentelemetry = { workspace = true }
 itertools = { workspace = true }
 libc = "0.2.153"
 log = { workspace = true }
-logforth = { version = "0.11", git = "http://github.com/andylokandy/logforth", rev = "0ca61ca", features = [
+logforth = { version = "0.12", features = [
     'json',
     'rolling_file',
     'opentelemetry',
@@ -32,6 +33,7 @@ opentelemetry = { workspace = true }
 opentelemetry-otlp = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 strip-ansi-escapes = "0.2"
 tonic = { workspace = true }
 

src/common/tracing/src/init.rs

@@ -224,15 +224,19 @@ pub fn init_logging(
         let labels = labels
             .iter()
             .chain(&cfg.otlp.endpoint.labels)
-            .map(|(k, v)| (k.clone().into(), v.clone().into()))
-            .chain([("category".into(), "system".into())]);
-        let otel = logforth::append::OpentelemetryLog::new(
+            .map(|(k, v)| (Cow::from(k.clone()), Cow::from(v.clone())))
+            .chain([(Cow::from("category"), Cow::from("system"))]);
+        let mut otel_builder = logforth::append::opentelemetry::OpentelemetryLogBuilder::new(
             log_name,
-            &cfg.otlp.endpoint.endpoint,
-            cfg.otlp.endpoint.protocol.into(),
-            labels,
+            format!("{}/v1/logs", &cfg.otlp.endpoint.endpoint),
         )
-        .expect("initialize opentelemetry logger");
+        .with_protocol(cfg.otlp.endpoint.protocol.into());
+        for (k, v) in labels {
+            otel_builder = otel_builder.add_label(k, v);
+        }
+        let otel = otel_builder
+            .build()
+            .expect("initialize opentelemetry logger");
         let dispatch = Dispatch::new()
             .filter(TargetFilter::level_for(
                 "databend::log::query",
@@ -290,23 +294,26 @@ pub fn init_logging(
                     "databend::log::query",
                     LevelFilter::Off,
                 ))
-                .layout(get_layout(&cfg.file.format))
                 .append(query_log_file);
             logger = logger.dispatch(dispatch);
         }
         if let Some(endpoint) = &cfg.query.otlp {
             let labels = labels
                 .iter()
                 .chain(&endpoint.labels)
-                .map(|(k, v)| (k.clone().into(), v.clone().into()))
-                .chain([("category".into(), "query".into())]);
-            let otel = logforth::append::OpentelemetryLog::new(
+                .map(|(k, v)| (Cow::from(k.clone()), Cow::from(v.clone())))
+                .chain([(Cow::from("category"), Cow::from("query"))]);
+            let mut otel_builder = logforth::append::opentelemetry::OpentelemetryLogBuilder::new(
                 log_name,
-                &endpoint.endpoint,
-                endpoint.protocol.into(),
-                labels,
+                format!("{}/v1/logs", &endpoint.endpoint),
             )
-            .expect("initialize opentelemetry logger");
+            .with_protocol(endpoint.protocol.into());
+            for (k, v) in labels {
+                otel_builder = otel_builder.add_label(k, v);
+            }
+            let otel = otel_builder
+                .build()
+                .expect("initialize opentelemetry logger");
             let dispatch = Dispatch::new()
                 .filter(TargetFilter::level_for_not(
                     "databend::log::query",
@@ -329,23 +336,26 @@ pub fn init_logging(
                     "databend::log::profile",
                     LevelFilter::Off,
                 ))
-                .layout(get_layout(&cfg.file.format))
                 .append(profile_log_file);
             logger = logger.dispatch(dispatch);
         }
         if let Some(endpoint) = &cfg.profile.otlp {
             let labels = labels
                 .iter()
                 .chain(&endpoint.labels)
-                .map(|(k, v)| (k.clone().into(), v.clone().into()))
-                .chain([("category".into(), "profile".into())]);
-            let otel = logforth::append::OpentelemetryLog::new(
+                .map(|(k, v)| (Cow::from(k.clone()), Cow::from(v.clone())))
+                .chain([(Cow::from("category"), Cow::from("profile"))]);
+            let mut otel_builder = logforth::append::opentelemetry::OpentelemetryLogBuilder::new(
                 log_name,
-                &endpoint.endpoint,
-                endpoint.protocol.into(),
-                labels,
+                format!("{}/v1/logs", &endpoint.endpoint),
             )
-            .expect("initialize opentelemetry logger");
+            .with_protocol(endpoint.protocol.into());
+            for (k, v) in labels {
+                otel_builder = otel_builder.add_label(k, v);
+            }
+            let otel = otel_builder
+                .build()
+                .expect("initialize opentelemetry logger");
             let dispatch = Dispatch::new()
                 .filter(TargetFilter::level_for_not(
                     "databend::log::profile",