MB-24898: Allow plasma to be close source, take #2

Change-Id: Ieaa95c5a0e4a9cf4ab6d9b1f440cde589a114491
couchbase · Jul 11, 2017 · 9cb7be2 · 9cb7be2
1 parent 801ffee
commit 9cb7be2
Show file tree

Hide file tree

Showing 17 changed files with 221 additions and 57 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -4,27 +4,27 @@ PROJECT (indexing)
 INCLUDE (FindCouchbaseGo)
 INCLUDE (FindCouchbaseJemalloc)
 
-get_filename_component(JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY)
-
-SET (_ldflags)
+IF (BUILD_ENTERPRISE)
+  MESSAGE(STATUS "Building EE edition of GSI")
+  SET (TAGS "jemalloc")
+ELSE (BUILD_ENTERPRISE)
+  MESSAGE(STATUS "Building CE edition of GSI")
+  SET (TAGS "jemalloc community")
+ENDIF (BUILD_ENTERPRISE)
+
+SET (LDFLAGS)
 IF (APPLE)
-  # On OS X 10.11 (El Capitan) upwards we can no longer use DYLD_LIBRARY_PATH to locate
-  # runtime dependancies. Instead add the location of forestdb etc to the actual
-  # binary itself.
-
-  # TODO: Spock GA
-  # Temporarily adding -s flag to allow 1.7.3 to work on OSX 10.12.4
-  # Remove -s flag (which strips debug info) when we move to 1.8.1
-  # See https://github.com/golang/go/issues/19734
-
-  SET (_ldflags "-s -extldflags '-Wl,-rpath,@executable_path/../lib'")
+  # On macOS, we must use -rpath to locate libraries
+  # TODO: Remove debug info stripping (-s) added to allow 1.7.3 to work on macOS
+  SET (LDFLAGS "-s -extldflags '-Wl,-rpath,@executable_path/../lib'")
 ENDIF ()
 
 IF (WIN32)
-  SET (_ldflags "${_ldflags} -linkmode internal")
+  SET (LDFLAGS "${LDFLAGS} -linkmode internal")
 ENDIF ()
 
 GET_FILENAME_COMPONENT (CURL_LIBRARY_DIR "${CURL_LIBRARIES}" DIRECTORY)
+GET_FILENAME_COMPONENT (JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY)
 
 SET(CGO_INCLUDE_DIRS "${FORESTDB_INCLUDE_DIR};${sigar_SOURCE_DIR}/include;${Platform_SOURCE_DIR}/include;${CURL_INCLUDE_DIR}")
 SET(CGO_LIBRARY_DIRS "${FORESTDB_LIBRARY_DIR};${sigar_BINARY_DIR}/src;${Platform_BINARY_DIR};${CURL_LIBRARY_DIR}")
@@ -36,15 +36,14 @@ ENDIF(NOT JEMALLOC_FOUND)
 SET (ENV{CGO_CFLAGS} "$ENV{CGO_CFLAGS} -DJEMALLOC=1")
 SET (CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS};${JEMALLOC_INCLUDE_DIR}")
 SET (CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS};${JEMALLOC_LIB_DIR}")
-SET (TAGS "jemalloc")
 
 GoInstall (TARGET indexer PACKAGE github.com/couchbase/indexing/secondary/cmd/indexer
   GOPATH "${PROJECT_SOURCE_DIR}/../../../.." "${GODEPSDIR}"
   DEPENDS forestdb
   CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
   CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
   GOTAGS "${TAGS}"
-  LDFLAGS "${_ldflags}"
+  LDFLAGS "${LDFLAGS}"
   INSTALL_PATH bin OUTPUT indexer
   GOVERSION 1.7.3)
 
@@ -53,21 +52,21 @@ GoInstall (TARGET projector PACKAGE github.com/couchbase/indexing/secondary/cmd/
   CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
   CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
   GOTAGS "${TAGS}"
-  LDFLAGS "${_ldflags}"
+  LDFLAGS "${LDFLAGS}"
   INSTALL_PATH bin
   GOVERSION 1.7.3)
 
 GoInstall (TARGET cbindex PACKAGE github.com/couchbase/indexing/secondary/cmd/cbindex
   GOPATH "${PROJECT_SOURCE_DIR}/../../../.." "${GODEPSDIR}"
   GOTAGS "${TAGS}"
-  LDFLAGS "${_ldflags}"
+  LDFLAGS "${LDFLAGS}"
   INSTALL_PATH bin
   GOVERSION 1.7.3)
 
 GoInstall (TARGET cbindexperf PACKAGE github.com/couchbase/indexing/secondary/cmd/cbindexperf
   GOPATH "${PROJECT_SOURCE_DIR}/../../../.." "${GODEPSDIR}"
   GOTAGS "${TAGS}"
-  LDFLAGS "${_ldflags}"
+  LDFLAGS "${LDFLAGS}"
   INSTALL_PATH bin
   GOVERSION 1.7.3)
 
@@ -77,15 +76,18 @@ GoInstall (TARGET cbindexplan PACKAGE github.com/couchbase/indexing/secondary/cm
   CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
   CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
   GOTAGS "${TAGS}"
-  LDFLAGS "${_ldflags}"
+  LDFLAGS "${LDFLAGS}"
   INSTALL_PATH bin OUTPUT cbindexplan
   GOVERSION 1.7.3)
 
-GoInstall(TARGET plasma_dump PACKAGE github.com/couchbase/nitro/cmd/plasma_dump
-  GOPATH "${CMAKE_SOURCE_DIR}/goproj" "${GODEPSDIR}"
-  CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
-  CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
-  GOTAGS "${TAGS}"
-  LDFLAGS "${_ldflags}"
-  INSTALL_PATH bin OUTPUT plasma_dump
-  GOVERSION 1.7.3)
+IF (BUILD_ENTERPRISE)
+  GoInstall(TARGET plasma_dump PACKAGE github.com/couchbase/nitro/cmd/plasma_dump
+    GOPATH "${CMAKE_SOURCE_DIR}/goproj" "${GODEPSDIR}"
+    CGO_INCLUDE_DIRS "${CGO_INCLUDE_DIRS}"
+    CGO_LIBRARY_DIRS "${CGO_LIBRARY_DIRS}"
+    GOTAGS "${TAGS}"
+    LDFLAGS "${LDFLAGS}"
+    INSTALL_PATH bin OUTPUT plasma_dump
+    GOVERSION 1.7.3)
+ENDIF (BUILD_ENTERPRISE)
+
diff --git a/secondary/build.sh b/secondary/build.sh
@@ -3,7 +3,7 @@
 if [ "$C_INCLUDE_PATH" == "" ]; then
   top="`pwd`/../../../../../.."
   export GOPATH="$top/goproj:$top/godeps"
-  export C_INCLUDE_PATH="$top/install/platform/include:$top/install/include:$top/forestdb/include:$top/install/build/tlm/deps/curl.exploded/include"
+  export C_INCLUDE_PATH="$top/install/platform/include:$top/install/include:$top/forestdb/include:$top/install/build/tlm/deps/curl.exploded/include:$top/sigar/include"
   export CGO_LDFLAGS="-L $top/install/lib"
   export LD_LIBRARY_PATH="$top/install/lib"
 fi
@@ -50,11 +50,13 @@ clean_projector(){
 
 build_protobuf(){
 
-    echo "Building Protobuf..."
-    cd protobuf
-    make
-    cd ..
-    echo "Done"
+    if which protoc; then
+      echo "Building Protobuf..."
+      cd protobuf
+      make
+      cd ..
+      echo "Done"
+    fi
 }
 
 if [ -z "$1" ]

diff --git a/secondary/common/index.go b/secondary/common/index.go
@@ -435,8 +435,7 @@ func (s IndexSnapType) String() string {
 
 }
 
-//NOTE: This type needs to be in sync with
-//smStrMap in index/global.go
+//NOTE: This type needs to be in sync with smStrMap
 type IndexType string
 
 const (

diff --git a/secondary/common/storage_mode.go b/secondary/common/storage_mode.go
@@ -12,6 +12,9 @@ package common
 import (
 	"strings"
 	"sync"
+
+	"github.com/couchbase/indexing/secondary/logging"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/plasma"
 )
 
 type StorageMode byte
@@ -28,11 +31,11 @@ func (s StorageMode) String() string {
 	case NOT_SET:
 		return "not_set"
 	case MOI:
-		return "memory_optimized"
+		return MemoryOptimized
 	case FORESTDB:
-		return "forestdb"
+		return ForestDB
 	case PLASMA:
-		return "plasma"
+		return PlasmaDB
 	default:
 		return "invalid"
 	}
@@ -41,10 +44,10 @@ func (s StorageMode) String() string {
 //NOTE: This map needs to be in sync with IndexType in
 //common/index.go
 var smStrMap = map[string]StorageMode{
-	"memdb":            MOI,
-	"memory_optimized": MOI,
-	"forestdb":         FORESTDB,
-	"plasma":           PLASMA,
+	MemDB:           MOI,
+	MemoryOptimized: MOI,
+	ForestDB:        FORESTDB,
+	PlasmaDB:        PLASMA,
 }
 
 //Global Storage Mode
@@ -64,7 +67,10 @@ func SetStorageMode(mode StorageMode) {
 	smLock.Lock()
 	defer smLock.Unlock()
 	gStorageMode = mode
-
+	if gStorageMode == PLASMA && !plasma.UsePlasma() {
+		logging.Warnf("Plasma is available only in EE but this is CE. Using ForestDB")
+		gStorageMode = FORESTDB
+	}
 }
 
 func SetStorageModeStr(mode string) bool {
@@ -73,6 +79,10 @@ func SetStorageModeStr(mode string) bool {
 	defer smLock.Unlock()
 	if s, ok := smStrMap[strings.ToLower(mode)]; ok {
 		gStorageMode = s
+		if gStorageMode == PLASMA && !plasma.UsePlasma() {
+			logging.Warnf("Plasma is available only in EE but this is CE. Using ForestDB")
+			gStorageMode = FORESTDB
+		}
 		return true
 	} else {
 		gStorageMode = NOT_SET

diff --git a/secondary/indexer/indexer.go b/secondary/indexer/indexer.go
@@ -34,8 +34,8 @@ import (
 	"github.com/couchbase/indexing/secondary/memdb"
 	"github.com/couchbase/indexing/secondary/memdb/nodetable"
 	projClient "github.com/couchbase/indexing/secondary/projector/client"
-	"github.com/couchbase/nitro/mm"
-	"github.com/couchbase/nitro/plasma"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/plasma"
 )
 
 type Indexer interface {

diff --git a/secondary/indexer/memdb_slice_impl.go b/secondary/indexer/memdb_slice_impl.go
@@ -33,7 +33,7 @@ import (
 	"github.com/couchbase/indexing/secondary/memdb"
 	"github.com/couchbase/indexing/secondary/memdb/nodetable"
 	"github.com/couchbase/indexing/secondary/memdb/skiplist"
-	"github.com/couchbase/nitro/mm"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
 )
 
 const (

diff --git a/secondary/indexer/plasma_community.go b/secondary/indexer/plasma_community.go
@@ -0,0 +1,22 @@
+// +build community
+
+package indexer
+
+// Copyright (c) 2014 Couchbase, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+// except in compliance with the License. You may obtain a copy of the License at
+//   http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed under the
+// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing permissions
+// and limitations under the License.
+
+import (
+	"github.com/couchbase/indexing/secondary/common"
+)
+
+func NewPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
+	idxInstId common.IndexInstId, isPrimary bool,
+	sysconf common.Config, idxStats *IndexStats) (Slice, error) {
+	panic("Plasma is only supported in Enterprise Edition")
+}
diff --git a/secondary/indexer/plasma_enterprise.go b/secondary/indexer/plasma_enterprise.go
@@ -0,0 +1,24 @@
+// +build !community
+
+package indexer
+
+// Copyright (c) 2014 Couchbase, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+// except in compliance with the License. You may obtain a copy of the License at
+//   http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed under the
+// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+// either express or implied. See the License for the specific language governing permissions
+// and limitations under the License.
+
+import (
+	"github.com/couchbase/indexing/secondary/common"
+)
+
+func NewPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
+	idxInstId common.IndexInstId, isPrimary bool,
+	sysconf common.Config, idxStats *IndexStats) (*plasmaSlice, error) {
+	return newPlasmaSlice(path, sliceId,
+		idxDefn, idxInstId, isPrimary,
+		sysconf, idxStats)
+}
diff --git a/secondary/indexer/plasma_slice.go b/secondary/indexer/plasma_slice.go
@@ -1,3 +1,7 @@
+// +build !community
+
+package indexer
+
 // Copyright (c) 2014 Couchbase, Inc.
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 // except in compliance with the License. You may obtain a copy of the License at
@@ -7,8 +11,6 @@
 // either express or implied. See the License for the specific language governing permissions
 // and limitations under the License.
 
-package indexer
-
 import (
 	"bytes"
 	"encoding/binary"
@@ -91,7 +93,7 @@ type plasmaSlice struct {
 	hasPersistence bool
 }
 
-func NewPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
+func newPlasmaSlice(path string, sliceId SliceId, idxDefn common.IndexDefn,
 	idxInstId common.IndexInstId, isPrimary bool,
 	sysconf common.Config, idxStats *IndexStats) (*plasmaSlice, error) {
 

diff --git a/secondary/indexer/settings.go b/secondary/indexer/settings.go
@@ -13,13 +13,15 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+
 	"github.com/couchbase/cbauth"
 	"github.com/couchbase/cbauth/metakv"
 	"github.com/couchbase/indexing/secondary/common"
 	"github.com/couchbase/indexing/secondary/logging"
 	"github.com/couchbase/indexing/secondary/pipeline"
-	"github.com/couchbase/nitro/mm"
-	"github.com/couchbase/nitro/plasma"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/plasma"
+
 	"io/ioutil"
 	"net/http"
 	"os"

diff --git a/secondary/indexer/stats_manager.go b/secondary/indexer/stats_manager.go
@@ -22,7 +22,7 @@ import (
 	"github.com/couchbase/indexing/secondary/common"
 	"github.com/couchbase/indexing/secondary/logging"
 	"github.com/couchbase/indexing/secondary/stats"
-	"github.com/couchbase/nitro/mm"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
 )
 
 var uptime time.Time

diff --git a/secondary/memdb/memdb.go b/secondary/memdb/memdb.go
@@ -6,8 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/couchbase/indexing/secondary/memdb/skiplist"
-	"github.com/couchbase/nitro/mm"
 	"io"
 	"io/ioutil"
 	"math"
@@ -19,6 +17,9 @@ import (
 	"sync/atomic"
 	"time"
 	"unsafe"
+
+	"github.com/couchbase/indexing/secondary/memdb/skiplist"
+	"github.com/couchbase/indexing/secondary/stubs/nitro/mm"
 )
 
 var version = 1
@@ -1174,5 +1175,5 @@ func MemoryInUse() (sz int64) {
 
 func Debug(flag bool) {
 	skiplist.Debug = flag
-	mm.Debug = flag
+	*mm.Debug = flag
 }
diff --git a/secondary/memdb/memdb_test.go b/secondary/memdb/memdb_test.go
@@ -9,7 +9,7 @@ import "math/rand"
 import "sync"
 import "runtime"
 import "encoding/binary"
-import "github.com/couchbase/nitro/mm"
+import "github.com/couchbase/indexing/secondary/stubs/nitro/mm"
 
 var testConf Config
 

diff --git a/secondary/stubs/nitro/mm/mm_community.go b/secondary/stubs/nitro/mm/mm_community.go
@@ -0,0 +1,21 @@
+// +build community
+
+package mm
+
+import "github.com/couchbase/indexing/secondary/memdb/skiplist"
+
+var Malloc skiplist.MallocFn
+var Free skiplist.FreeFn
+
+var Debug *bool = &[]bool{false}[0]
+
+func FreeOSMemory() {
+}
+
+func Size() uint64 {
+	return 0
+}
+
+func Stats() string {
+	return ""
+}