Skip to content

Commit

Permalink
Update faiss commit and resolve conflict (opensearch-project#1443)
Browse files Browse the repository at this point in the history
Signed-off-by: Heemin Kim <[email protected]>
  • Loading branch information
heemin32 authored Feb 13, 2024
1 parent 06ccdbc commit ced1ff2
Show file tree
Hide file tree
Showing 20 changed files with 1,041 additions and 1,083 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Infrastructure
### Documentation
### Maintenance
* Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443)
### Refactoring
15 changes: 5 additions & 10 deletions jni/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -181,17 +181,15 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S
${TARGET_LIB_FAISS} SHARED
${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_FaissService.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_wrapper.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/utils/BitSet.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollector.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp)
${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_util.cpp
)
target_link_libraries(${TARGET_LIB_FAISS} ${TARGET_LINK_FAISS_LIB} ${TARGET_LIB_COMMON} OpenMP::OpenMP_CXX)
target_include_directories(${TARGET_LIB_FAISS} PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss
${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss/utils
$ENV{JAVA_HOME}/include
$ENV{JAVA_HOME}/include/${JVM_OS_TYPE}
${CMAKE_CURRENT_SOURCE_DIR}/external/faiss)
${CMAKE_CURRENT_SOURCE_DIR}/external/faiss
)
set_target_properties(${TARGET_LIB_FAISS} PROPERTIES SUFFIX ${LIB_EXT})
set_target_properties(${TARGET_LIB_FAISS} PROPERTIES POSITION_INDEPENDENT_CODE ON)

Expand Down Expand Up @@ -228,12 +226,9 @@ if ("${WIN32}" STREQUAL "")
add_executable(
jni_test
tests/faiss_wrapper_test.cpp
tests/faiss_util_test.cpp
tests/nmslib_wrapper_test.cpp
tests/test_util.cpp
tests/knn_extension/faiss/utils/BitSetTest.cpp
tests/knn_extension/faiss/utils/HeapTest.cpp
tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp
tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp
)

target_link_libraries(
Expand Down
2 changes: 1 addition & 1 deletion jni/external/faiss
Submodule faiss updated 38 files
+1 −0 CHANGELOG.md
+2 −0 benchs/link_and_code/README.md
+1 −0 contrib/evaluation.py
+17 −10 faiss/IndexAdditiveQuantizer.cpp
+2 −2 faiss/IndexAdditiveQuantizerFastScan.cpp
+13 −10 faiss/IndexBinaryHNSW.cpp
+81 −150 faiss/IndexFastScan.cpp
+9 −8 faiss/IndexFastScan.h
+64 −284 faiss/IndexHNSW.cpp
+7 −45 faiss/IndexHNSW.h
+7 −11 faiss/IndexIVFAdditiveQuantizerFastScan.cpp
+1 −2 faiss/IndexIVFAdditiveQuantizerFastScan.h
+412 −466 faiss/IndexIVFFastScan.cpp
+44 −37 faiss/IndexIVFFastScan.h
+1 −1 faiss/IndexIVFPQ.cpp
+8 −8 faiss/IndexIVFPQFastScan.cpp
+1 −2 faiss/IndexIVFPQFastScan.h
+0 −1 faiss/impl/AuxIndexStructures.h
+41 −45 faiss/impl/HNSW.cpp
+7 −8 faiss/impl/HNSW.h
+1 −1 faiss/impl/Quantizer.h
+224 −168 faiss/impl/ResultHandler.h
+3 −0 faiss/impl/pq4_fast_scan.cpp
+7 −6 faiss/impl/pq4_fast_scan.h
+51 −37 faiss/impl/pq4_fast_scan_search_1.cpp
+37 −46 faiss/impl/pq4_fast_scan_search_qbs.cpp
+374 −202 faiss/impl/simd_result_handlers.h
+8 −0 faiss/python/swigfaiss.swig
+59 −48 faiss/utils/distances.cpp
+4 −4 faiss/utils/distances_fused/avx512.cpp
+1 −1 faiss/utils/distances_fused/avx512.h
+1 −1 faiss/utils/distances_fused/distances_fused.cpp
+1 −1 faiss/utils/distances_fused/distances_fused.h
+4 −4 faiss/utils/distances_fused/simdlib_based.cpp
+1 −1 faiss/utils/distances_fused/simdlib_based.h
+74 −5 tests/test_fast_scan_ivf.py
+426 −0 tests/test_graph_based.py
+0 −400 tests/test_index.py
25 changes: 25 additions & 0 deletions jni/include/faiss_util.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: Apache-2.0
//
// The OpenSearch Contributors require contributions made to
// this file be licensed under the Apache-2.0 license or a
// compatible open source license.
//
// Modifications Copyright OpenSearch Contributors. See
// GitHub history for details.

/**
* This file contains util methods which are free of JNI to be used in faiss_wrapper.cpp
*/

#ifndef OPENSEARCH_KNN_FAISS_UTIL_H
#define OPENSEARCH_KNN_FAISS_UTIL_H

#include "faiss/impl/IDGrouper.h"
#include <memory>

namespace faiss_util {
std::unique_ptr<faiss::IDGrouperBitmap> buildIDGrouperBitmap(int *parentIdsArray, int parentIdsLength, std::vector<uint64_t>* bitmap);
};


#endif //OPENSEARCH_KNN_FAISS_UTIL_H
69 changes: 0 additions & 69 deletions jni/include/knn_extension/faiss/MultiVectorResultCollector.h

This file was deleted.

This file was deleted.

54 changes: 0 additions & 54 deletions jni/include/knn_extension/faiss/utils/BitSet.h

This file was deleted.

Loading

0 comments on commit ced1ff2

Please sign in to comment.