From 8645184cabcfc5523115ca7d2ec7e4f75d16af8e Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Sun, 26 Mar 2023 22:55:25 -0400 Subject: [PATCH] New build profile that uses HBase 2.5 libraries (#6) * New build profile that uses HBase 2.5 libraries * Rename profiles and remove duplicate source code --------- Co-authored-by: Charles Connell Co-authored-by: Junegunn Choi --- Makefile | 2 +- README.md | 5 +- project.clj | 29 +++++--- .../hbase_region_inspector/hbase/impl.clj | 6 ++ .../hbase_region_inspector/hbase/impl.clj | 66 +++++++++---------- src/main/hbase_region_inspector/hbase.clj | 6 +- 6 files changed, 65 insertions(+), 49 deletions(-) rename src/{hbase-cdh4 => cdh4}/hbase_region_inspector/hbase/impl.clj (97%) rename src/{hbase-1.0 => hbase1}/hbase_region_inspector/hbase/impl.clj (59%) diff --git a/Makefile b/Makefile index c5e7bed..9c528c1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -profile?=1.0 +profile?=hbase1 all: build release diff --git a/README.md b/README.md index e460584..f9e6c03 100644 --- a/README.md +++ b/README.md @@ -62,11 +62,14 @@ brew install npm leiningen ### Build ```sh -# For HBase 0.98 and above +# For HBase 0.98 and above (profile=hbase1) make # HBase 0.94 (CDH4) make profile=cdh4 + +# Build with HBase 2 dependencies +make profile=hbase2 ``` ## License diff --git a/project.clj b/project.clj index 66e916b..a22054b 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,5 @@ (def project-version "0.3.8") -(def current :1.0) +(def current :hbase1) (defn bin [profile] (str "hbase-region-inspector-" project-version (when (not= profile current) (str "-" (name profile))))) @@ -49,7 +49,7 @@ :jar-name ~(jar :cdh4) :uberjar-name ~(jar :cdh4 "-standalone") :target-path "target/cdh4" - :source-paths ["src/hbase-cdh4"] + :source-paths ["src/cdh4"] ;; lein with-profile cdh4 deps :tree :dependencies [[org.apache.hbase/hbase "0.94.15-cdh4.7.1" :exclusions @@ -59,13 +59,26 @@ [javax.xml.bind/jaxb-api org.slf4j/slf4j-api org.slf4j/slf4j-log4j12]] [org.slf4j/slf4j-api "1.7.12"] [org.slf4j/slf4j-log4j12 "1.7.12"]]} - :1.0-test {:dependencies [[org.apache.hbase/hbase-testing-util "1.0.0"]]} - :1.0 ^:leaky {:bin {:name ~(bin :1.0)} - :jar-name ~(jar :1.0) - :uberjar-name ~(jar :1.0 "-standalone") - :target-path "target/1.0" - :source-paths ["src/hbase-1.0"] + :hbase1-test {:dependencies [[org.apache.hbase/hbase-testing-util "1.0.0"]]} + :hbase1 ^:leaky {:bin {:name ~(bin :hbase1)} + :jar-name ~(jar :hbase1) + :uberjar-name ~(jar :hbase1 "-standalone") + :target-path "target/hbase1" + :source-paths ["src/hbase1"] :dependencies [[org.apache.hbase/hbase-client "1.0.0"] [org.apache.hbase/hbase-common "1.0.0"] [org.apache.zookeeper/zookeeper "3.5.7"]]} + :hbase2-test {:dependencies [[org.apache.hbase/hbase-testing-util "2.5.3" + :exclusions [net.minidev/json-smart org.glassfish.web/javax.servlet.jsp]]]} + :hbase2 ^:leaky {:bin {:name ~(bin :hbase2)} + :jar-name ~(jar :hbase2) + :uberjar-name ~(jar :hbase2 "-standalone") + :target-path "target/hbase2" + ;; No difference between hbase1 and hbase2 + :source-paths ["src/hbase1"] + :dependencies [[org.apache.hbase/hbase-client "2.5.3" + :exclusions [net.minidev/json-smart]] + [org.apache.hbase/hbase-common "2.5.3" + :exclusions [net.minidev/json-smart]] + [org.apache.zookeeper/zookeeper "3.5.7"]]} :uberjar {:aot :all}}) diff --git a/src/hbase-cdh4/hbase_region_inspector/hbase/impl.clj b/src/cdh4/hbase_region_inspector/hbase/impl.clj similarity index 97% rename from src/hbase-cdh4/hbase_region_inspector/hbase/impl.clj rename to src/cdh4/hbase_region_inspector/hbase/impl.clj index 858852d..a37f2b5 100644 --- a/src/hbase-cdh4/hbase_region_inspector/hbase/impl.clj +++ b/src/cdh4/hbase_region_inspector/hbase/impl.clj @@ -5,6 +5,7 @@ (:import org.apache.hadoop.hbase.client.HConnectionManager org.apache.hadoop.hbase.client.HTable org.apache.hadoop.hbase.util.Bytes + org.apache.hadoop.hbase.client.HBaseAdmin org.apache.hadoop.hbase.HServerLoad org.apache.hadoop.hbase.HServerLoad$RegionLoad java.nio.ByteBuffer)) @@ -127,3 +128,8 @@ (for [region->info aggregated [k v] region->info] (assoc v :name (util/byte-array->str (.array k)))))) + +(defn connect-admin + "Creates HBaseAdmin instance with the given configuration." + [conf] + (HBaseAdmin. conf)) diff --git a/src/hbase-1.0/hbase_region_inspector/hbase/impl.clj b/src/hbase1/hbase_region_inspector/hbase/impl.clj similarity index 59% rename from src/hbase-1.0/hbase_region_inspector/hbase/impl.clj rename to src/hbase1/hbase_region_inspector/hbase/impl.clj index 5699d24..77e1b85 100644 --- a/src/hbase-1.0/hbase_region_inspector/hbase/impl.clj +++ b/src/hbase1/hbase_region_inspector/hbase/impl.clj @@ -3,12 +3,10 @@ [hbase-region-inspector.util :as util] [hbase-region-inspector.hbase.base :as base]) (:import [org.apache.hadoop.hbase - util.Bytes client.Admin + util.Bytes client.Admin client.ConnectionFactory HRegionInfo RegionLoad ClusterStatus ServerName ServerLoad] java.nio.ByteBuffer)) -;; http://archive.cloudera.com/cdh5/cdh/5/hbase-0.98.6-cdh5.3.3/apidocs/index.html - (defn info->map "Builds map from HRegionInfo" [^HRegionInfo info] @@ -18,42 +16,35 @@ (defn load->map "Builds map from RegionLoad" [^RegionLoad load] - (let [base {:compacted-kvs (.getCurrentCompactedKVs load) - :memstore-size-mb (.getMemStoreSizeMB load) - :read-requests (.getReadRequestsCount load) - :requests (.getRequestsCount load) - :root-index-size-kb (.getRootIndexSizeKB load) - :store-file-index-size-mb (.getStorefileIndexSizeMB load) - :store-files (.getStorefiles load) - :store-file-size-mb (.getStorefileSizeMB load) - :stores (.getStores load) - ; :store-uncompressed-size-mb - :total-compacting-kvs (.getTotalCompactingKVs load) - :bloom-size-kb (.getTotalStaticBloomSizeKB load) - :total-index-size-kb (.getTotalStaticIndexSizeKB load) - :write-requests (.getWriteRequestsCount load)} - loc (.getDataLocality load)] - (assoc base - :store-uncompressed-size-mb (.getStoreUncompressedSizeMB load) - :locality (* 100 loc) - :local-size-mb (* loc (:store-file-size-mb base))))) + (let [loc (.getDataLocality load) + store-file-size-mb (.getStorefileSizeMB load)] + {:compacted-kvs (.getCurrentCompactedKVs load) + :memstore-size-mb (.getMemStoreSizeMB load) + :read-requests (.getReadRequestsCount load) + :requests (.getRequestsCount load) + :root-index-size-kb (.getRootIndexSizeKB load) + :store-file-index-size-mb (.getStorefileIndexSizeMB load) + :store-files (.getStorefiles load) + :store-file-size-mb store-file-size-mb + :stores (.getStores load) + :store-uncompressed-size-mb (.getStoreUncompressedSizeMB load) + :total-compacting-kvs (.getTotalCompactingKVs load) + :bloom-size-kb (.getTotalStaticBloomSizeKB load) + :total-index-size-kb (.getTotalStaticIndexSizeKB load) + :write-requests (.getWriteRequestsCount load) + :locality (* 100 loc) + :local-size-mb (* loc store-file-size-mb)})) (defn server-load->map "Transforms ServerLoad object into clojure map" [^ServerLoad load] - (assoc - {:max-heap-mb (.getMaxHeapMB load) - :used-heap-mb (.getUsedHeapMB load) - :regions (.getNumberOfRegions load) - :requests-rate (.getNumberOfRequests load) - :store-files (.getStorefiles load) - :store-file-size-mb (.getStorefileSizeInMB load)} - :store-uncompressed-size-mb - (some->> load - str - (re-find #"storefileUncompressedSizeMB=([0-9]+)") - last - Integer/parseInt))) + {:max-heap-mb (.getMaxHeapMB load) + :used-heap-mb (.getUsedHeapMB load) + :regions (.getNumberOfRegions load) + :requests-rate (.getNumberOfRequests load) + :store-files (.getStorefiles load) + :store-file-size-mb (.getStorefileSizeInMB load) + :store-uncompressed-size-mb (.getStoreUncompressedSizeMB load)}) (defn- online-regions "Retrieves the information of online regions using HBaseAdmin.getOnlineRegions" @@ -101,3 +92,8 @@ (for [region->info aggregated [k v] region->info] (assoc v :name (util/byte-array->str (.array ^ByteBuffer k)))))) + +(defn connect-admin + "Creates HBaseAdmin instance with the given configuration." + [conf] + (.getAdmin (ConnectionFactory/createConnection conf))) diff --git a/src/main/hbase_region_inspector/hbase.clj b/src/main/hbase_region_inspector/hbase.clj index 1fa901d..aa9ae12 100644 --- a/src/main/hbase_region_inspector/hbase.clj +++ b/src/main/hbase_region_inspector/hbase.clj @@ -93,11 +93,9 @@ (.getInt (build-hbase-conf conf) "hbase.regionserver.info.port" 60030)) (defn connect-admin - "Creates HBaseAdmin instance with the given configuration. - TODO: HBaseAdmin is deprecated in favor of Admin in the recent versions of - HBase" + "Creates HBaseAdmin instance with the given configuration." [conf] - (HBaseAdmin. (build-hbase-conf conf))) + (hbase-impl/connect-admin (build-hbase-conf conf))) (defmacro with-admin "Evaluates body with HBaseAdmin created with conf bound to name and