Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GLUTEN-1632][CH]Daily Update Clickhouse Version (20240906) #7137

Merged
merged 7 commits into from
Sep 6, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ class GlutenClickHouseMergeTreeCacheDataSSuite
.set(
"spark.gluten.sql.columnar.backend.ch.runtime_settings.mergetree.merge_after_insert",
"false")
// .set("spark.gluten.sql.columnar.backend.ch.runtime_config.path", "/data") // for local test
}

override protected def beforeEach(): Unit = {
Expand Down
4 changes: 2 additions & 2 deletions cpp-ch/clickhouse.version
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
CH_BRANCH=rebase_ch/20240830
CH_COMMIT=5e2eaab52ac
CH_BRANCH=rebase_ch/20240906
CH_COMMIT=4513a954b95
3 changes: 1 addition & 2 deletions cpp-ch/local-engine/Parser/MergeTreeRelParser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,7 @@ DB::QueryPlanPtr MergeTreeRelParser::parseReadRel(

auto read_step = storage->reader.readFromParts(
selected_parts,
/* alter_conversions = */
{},
storage->getMutationsSnapshot({}),
names_and_types_list.getNames(),
storage_snapshot,
*query_info,
Expand Down
3 changes: 1 addition & 2 deletions cpp-ch/local-engine/Storages/Cache/CacheManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,7 @@ Task CacheManager::cachePart(const MergeTreeTableInstance & table, const MergeTr
= StorageMergeTreeFactory::getDataPartsByNames(storage->getStorageID(), "", {job_detail.table.parts.front().name});
auto read_step = storage->reader.readFromParts(
selected_parts,
/* alter_conversions = */
{},
storage->getMutationsSnapshot({}),
names_and_types_list.getNames(),
storage_snapshot,
*query_info,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,6 @@ RangesInDataParts MergeTreeTableInstance::extractRange(DataPartsVector parts_vec
ranges_in_data_part.data_part = name_index.at(part.name);
ranges_in_data_part.part_index_in_query = 0;
ranges_in_data_part.ranges.emplace_back(MarkRange(part.begin, part.end));
ranges_in_data_part.alter_conversions = std::make_shared<AlterConversions>();
return ranges_in_data_part;
});
return ranges_in_data_parts;
Expand Down
20 changes: 19 additions & 1 deletion cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,19 @@ class SparkStorageMergeTree : public MergeTreeData
{
friend class MergeSparkMergeTreeTask;

struct SparkMutationsSnapshot : public IMutationsSnapshot
{
SparkMutationsSnapshot() = default;

MutationCommands getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const override { return {}; }
std::shared_ptr<MergeTreeData::IMutationsSnapshot> cloneEmpty() const override
{
return std::make_shared<SparkMutationsSnapshot>();
}

NameSet getAllUpdatedColumns() const override { return {}; }
};

public:
static void wrapRangesInDataParts(DB::ReadFromMergeTree & source, const DB::RangesInDataParts & ranges);
static void analysisPartsByRanges(DB::ReadFromMergeTree & source, const DB::RangesInDataParts & ranges_in_data_parts);
Expand Down Expand Up @@ -94,8 +107,13 @@ class SparkStorageMergeTree : public MergeTreeData
void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr context) override;
void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr context) override;
bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override;
MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & /*part*/) const override { return {}; }
void attachRestoredParts(MutableDataPartsVector && /*parts*/) override { throw std::runtime_error("not implement"); }

public:
MutationsSnapshotPtr getMutationsSnapshot(const IMutationsSnapshot::Params & /*params*/) const override
{
return std::make_shared<SparkMutationsSnapshot>();
};
};

class SparkWriteStorageMergeTree final : public SparkStorageMergeTree
Expand Down
Loading