From 6fda2bd70b86499ac8df23241ee8cc904b4b4061 Mon Sep 17 00:00:00 2001 From: deysn Date: Tue, 19 Jun 2018 12:40:41 -0400 Subject: [PATCH 1/8] TWRPv2 added , removed some classes, DBFriendly Projection fixed. TWRPv2 is the working version where only the centroids and counts are stored, instead of the whole set of vectors. Fixed DBFriendly Projection. Removed some classes that were not used. --- .classpath | 18 +- src/main/java/edu/uc/rphash/RPHash.java | 53 +- .../edu/uc/rphash/RPHashAdaptive2Pass.java | 113 ++- .../rphash/RPHashAdaptive2PassParallel.java | 370 --------- .../java/edu/uc/rphash/RPHashMultiProj.java | 307 ------- .../java/edu/uc/rphash/RPHashStreamingAK.java | 198 ----- .../edu/uc/rphash/Readers/RPHashObject.java | 14 +- src/main/java/edu/uc/rphash/TWRP1.java | 763 ++++++++++++++++++ src/main/java/edu/uc/rphash/TWRPv2.java | 534 ++++++++++++ .../java/edu/uc/rphash/decoders/Golay.java | 350 -------- .../java/edu/uc/rphash/decoders/PsdLSH.java | 252 ------ .../KHHCentroidCounterPush.java | 56 -- .../edu/uc/rphash/knee/BiggestMergeKnee.java | 38 - .../edu/uc/rphash/knee/KneeAlgorithm.java | 7 - .../java/edu/uc/rphash/knee/LpointKnee.java | 66 -- .../java/edu/uc/rphash/knee/SimpleKnee.java | 25 - .../projections/DBFriendlyProjection.java | 20 + .../edu/uc/rphash/standardhash/CrapWow.java | 121 --- .../edu/uc/rphash/tests/ScalabilityTest.java | 130 --- .../java/edu/uc/rphash/tests/TestRPhash.java | 4 +- .../tests/clusterers/AdaptiveMeanShift.java | 432 ---------- .../tests/clusterers/Agglomerative.java | 151 ---- .../tests/clusterers/Agglomerative2.java | 372 --------- .../uc/rphash/tests/clusterers/Kmeans.java | 330 -------- .../clusterers/LloydIterativeKmeans.java | 250 ------ .../edu/uc/rphash/tests/clusterers/MLE2.java | 333 -------- .../clusterers/MaxLikelihoodKMeans2.java | 451 ----------- .../edu/uc/rphash/tests/clusterers/SVD.java | 456 ----------- .../uc/rphash/tests/testStreamingRPHash.java | 184 ----- 29 files changed, 1427 insertions(+), 4971 deletions(-) delete mode 100644 src/main/java/edu/uc/rphash/RPHashAdaptive2PassParallel.java delete mode 100644 src/main/java/edu/uc/rphash/RPHashMultiProj.java delete mode 100644 src/main/java/edu/uc/rphash/RPHashStreamingAK.java create mode 100644 src/main/java/edu/uc/rphash/TWRP1.java create mode 100644 src/main/java/edu/uc/rphash/TWRPv2.java delete mode 100644 src/main/java/edu/uc/rphash/decoders/Golay.java delete mode 100644 src/main/java/edu/uc/rphash/decoders/PsdLSH.java delete mode 100644 src/main/java/edu/uc/rphash/frequentItemSet/KHHCentroidCounterPush.java delete mode 100644 src/main/java/edu/uc/rphash/knee/BiggestMergeKnee.java delete mode 100644 src/main/java/edu/uc/rphash/knee/KneeAlgorithm.java delete mode 100644 src/main/java/edu/uc/rphash/knee/LpointKnee.java delete mode 100644 src/main/java/edu/uc/rphash/knee/SimpleKnee.java delete mode 100644 src/main/java/edu/uc/rphash/standardhash/CrapWow.java delete mode 100644 src/main/java/edu/uc/rphash/tests/ScalabilityTest.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/AdaptiveMeanShift.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative2.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/Kmeans.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/LloydIterativeKmeans.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/MLE2.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/MaxLikelihoodKMeans2.java delete mode 100644 src/main/java/edu/uc/rphash/tests/clusterers/SVD.java delete mode 100644 src/main/java/edu/uc/rphash/tests/testStreamingRPHash.java diff --git a/.classpath b/.classpath index 29836c6..8d805ce 100644 --- a/.classpath +++ b/.classpath @@ -1,9 +1,9 @@ - - - - - - - - - + + + + + + + + + diff --git a/src/main/java/edu/uc/rphash/RPHash.java b/src/main/java/edu/uc/rphash/RPHash.java index e62b867..2977bc4 100644 --- a/src/main/java/edu/uc/rphash/RPHash.java +++ b/src/main/java/edu/uc/rphash/RPHash.java @@ -22,11 +22,11 @@ import edu.uc.rphash.decoders.DepthProbingLSH; import edu.uc.rphash.decoders.Dn; import edu.uc.rphash.decoders.E8; -import edu.uc.rphash.decoders.Golay; + import edu.uc.rphash.decoders.Leech; import edu.uc.rphash.decoders.MultiDecoder; import edu.uc.rphash.decoders.OriginDecoder; -import edu.uc.rphash.decoders.PsdLSH; + import edu.uc.rphash.decoders.Spherical; import edu.uc.rphash.projections.DBFriendlyProjection; import edu.uc.rphash.projections.FJLTProjection; @@ -34,13 +34,13 @@ import edu.uc.rphash.projections.NoProjection; import edu.uc.rphash.projections.SVDProjection; import edu.uc.rphash.tests.StatTests; -import edu.uc.rphash.tests.clusterers.AdaptiveMeanShift; + import edu.uc.rphash.tests.clusterers.Agglomerative3; import edu.uc.rphash.tests.clusterers.DummyClusterer; import edu.uc.rphash.tests.clusterers.DBScan; import edu.uc.rphash.tests.clusterers.KMeans2; import edu.uc.rphash.tests.clusterers.KMeansPlusPlus; -import edu.uc.rphash.tests.clusterers.LloydIterativeKmeans; + import edu.uc.rphash.tests.clusterers.MultiKMPP; import edu.uc.rphash.tests.clusterers.StreamingKmeans; import edu.uc.rphash.tests.clusterers.StreamingKmeans2; @@ -552,11 +552,7 @@ public static List runConfigs(List untaggedArgs, so.setDecoderType(new E8(2f)); break; } - case "golay": { - o.setDecoderType(new Golay()); - so.setDecoderType(new Golay()); - break; - } + case "multie8": { o.setDecoderType(new MultiDecoder( o.getInnerDecoderMultiplier() * 8, new E8(2f))); @@ -576,23 +572,9 @@ public static List runConfigs(List untaggedArgs, .getInnerDecoderMultiplier() * 24, new Leech())); break; } - case "levypstable": { - o.setDecoderType(new PsdLSH(PsdLSH.LEVY, o.getDimparameter())); - so.setDecoderType(new PsdLSH(PsdLSH.LEVY, o.getDimparameter())); - break; - } - case "cauchypstable": { - o.setDecoderType(new PsdLSH(PsdLSH.CAUCHY, o.getDimparameter())); - so.setDecoderType(new PsdLSH(PsdLSH.CAUCHY, o.getDimparameter())); - break; - } - case "gaussianpstable": { - o.setDecoderType(new PsdLSH(PsdLSH.GAUSSIAN, o - .getDimparameter())); - so.setDecoderType(new PsdLSH(PsdLSH.GAUSSIAN, o - .getDimparameter())); - break; - } + + + case "sphere": {// pad to ~32 bits // int ctsofsphere = // (int)(Math.log(o.getDimparameter()*2)/Math.log(2.0)) /2; @@ -659,13 +641,7 @@ public static List runConfigs(List untaggedArgs, o.setOfflineClusterer(new KMeansPlusPlus()); so.setOfflineClusterer(new KMeansPlusPlus()); break; - case "adaptivemeanshift": { - - o.setOfflineClusterer(new AdaptiveMeanShift()); - so.setOfflineClusterer(new AdaptiveMeanShift()); - - break; - } + case "kmpp": { o.setOfflineClusterer(new KMeansPlusPlus()); @@ -727,11 +703,7 @@ public static List runConfigs(List untaggedArgs, runitems.add(new KMeans2(k, o.getRawData())); break; } - case "pkmeans": - runitems.add(new LloydIterativeKmeans(k, o.getRawData(), o - .getNumProjections())); - break; - + case "kmeansplusplus": runitems.add(new KMeansPlusPlus(o.getRawData(), k)); break; @@ -743,10 +715,7 @@ public static List runConfigs(List untaggedArgs, runitems.add(new StreamingKmeans2(o)); break; } - case "adaptivemeanshift": { - runitems.add(new AdaptiveMeanShift()); - break; - } + case "adaptive": { runitems.add(new RPHashAdaptive2Pass(o)); break; diff --git a/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java b/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java index 013ea27..1ff2008 100644 --- a/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java +++ b/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java @@ -1,8 +1,10 @@ package edu.uc.rphash; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -16,11 +18,12 @@ import edu.uc.rphash.tests.StatTests; import edu.uc.rphash.tests.clusterers.Agglomerative3; import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; public class RPHashAdaptive2Pass implements Clusterer, Runnable { - boolean znorm = true; + boolean znorm = false; private int counter; @@ -63,17 +66,21 @@ float[] medoid(List X) { //float[] rngvec; the range vector is moot if incoming data has been normalized //post normalization it should all be zero centered, with variance 1 - /* * super simple hash algorithm, reminiscient of pstable lsh */ + // xt is the projected vector and x is the original vector , rngvec is the randomly generated vector of projected dim. + public long hashvec(float[] xt, float[] x, HashMap> IDAndCent, HashMap> IDAndLabel,int ct) { - long s = 1;//fixes leading 0's bug + long s = 1; //fixes leading 0's bug for (int i = 0; i < xt.length; i++) { - s <<= 1; +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. if (xt[i] > rngvec[i]) - s += 1; +// s += 1; + s= s+1; + if (IDAndCent.containsKey(s)) { IDAndLabel.get(s).add(ct); IDAndCent.get(s).add(x); @@ -143,16 +150,17 @@ public List> findDensityModes() { projector.init(); int ct = 0; - if(znorm == true){ - float[] variance = StatTests.varianceCol(so.getRawData()); - float[] mean = StatTests.meanCols(so.getRawData()); - // #process data by adding to the counter - for (float[] x : so.getRawData()) - { - addtocounter(x, projector, IDAndCent,IDAndID,ct++,mean,variance); - } - } - else +// if(znorm == true){ +// float[] variance = StatTests.varianceCol(so.getRawData()); +// float[] mean = StatTests.meanCols(so.getRawData()); +// // #process data by adding to the counter +// for (float[] x : so.getRawData()) +// { +// addtocounter(x, projector, IDAndCent,IDAndID,ct++,mean,variance); +// } +// } +// +// else { for (float[] x : so.getRawData()) @@ -161,6 +169,34 @@ public List> findDensityModes() { } } + + for (Long name: IDAndCent.keySet()){ + + String key =name.toString(); + + + String value = IDAndCent.get(name).toString() ; + + // String value1 = Arrays.toString(value.toString()); + + System.out.println(key + " " + value); + + + + +} + + for (Long name: IDAndID.keySet()){ + + String key =name.toString(); + String value = IDAndID.get(name).toString(); + System.out.println(key + " " + value); + + +} + + + // next we want to prune the tree by parent count comparison // follows breadthfirst search HashMap denseSetOfIDandCount = new HashMap(); @@ -235,6 +271,7 @@ public void run() { centroids.add(medoid(clustermembers.get(i))); } Agglomerative3 aggloOffline = new Agglomerative3(centroids, so.getk()); + System.out.println(centroids.size()); aggloOffline.setWeights(weights); this.centroids = aggloOffline.getCentroids(); } @@ -242,26 +279,26 @@ public void run() { public static void main(String[] args) throws FileNotFoundException, IOException { - int k = 10; - int d = 1000; - int n = 10000; - float var = 0.1f; - int count = 10; - System.out.printf("ClusterVar\t"); - for (int i = 0; i < count; i++) - System.out.printf("Trial%d\t", i); - System.out.printf("RealWCSS\n"); - - for (float f = var; f < 5.01; f += .05f) { + int k = 3; + int d = 100; + int n = 2000; + float var = 1.0f;//0.5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + for (float f = var; f < 1.01; f += 1.5f) { float avgrealwcss = 0; float avgtime = 0; - System.out.printf("%f\t", f); + // System.out.printf("%f\t", f); for (int i = 0; i < count; i++) { - GenerateData gen = new GenerateData(k, n / k, d, f, true, .5f); + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); // gen.writeCSVToFile(new // File("/home/lee/Desktop/reclsh/in.csv")); RPHashObject o = new SimpleArrayReader(gen.data, k); - o.setDimparameter(32); + o.setDimparameter(4); RPHashAdaptive2Pass rphit = new RPHashAdaptive2Pass(o); long startTime = System.nanoTime(); List centsr = rphit.getCentroids(); @@ -271,12 +308,22 @@ public static void main(String[] args) throws FileNotFoundException, avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), gen.getData()); - System.out.printf("%.0f\t", - StatTests.WCSSECentroidsFloat(centsr, gen.data)); - System.gc(); + + String Output = "/C:/Users/user/Desktop/temp/OutputTwrpCents" ; + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + // System.out.printf("%.0f\t", + // StatTests.WCSSECentroidsFloat(centsr, gen.data)); + // System.gc(); } - System.out.printf("%.0f\n", avgrealwcss / count); + // System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + + } @Override diff --git a/src/main/java/edu/uc/rphash/RPHashAdaptive2PassParallel.java b/src/main/java/edu/uc/rphash/RPHashAdaptive2PassParallel.java deleted file mode 100644 index 4ad318a..0000000 --- a/src/main/java/edu/uc/rphash/RPHashAdaptive2PassParallel.java +++ /dev/null @@ -1,370 +0,0 @@ -package edu.uc.rphash; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.TreeSet; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.Readers.SimpleArrayReader; -import edu.uc.rphash.projections.Projector; -import edu.uc.rphash.tests.StatTests; -import edu.uc.rphash.tests.clusterers.Agglomerative3; -import edu.uc.rphash.tests.generators.GenerateData; - -public class RPHashAdaptive2PassParallel implements Clusterer, Runnable { - - boolean znorm = true; - - private float[] rngvec; - private List centroids = null; - private RPHashObject so; - int threads = 4; - - public RPHashAdaptive2PassParallel(RPHashObject so) { - this.threads = 4; - this.so = so; - } - - public RPHashAdaptive2PassParallel(List data, int k, int processors) { - this.threads = processors; - so = new SimpleArrayReader(data, k); - } - - public List getCentroids(RPHashObject so) { - this.so = so; - return getCentroids(); - } - - @Override - public List getCentroids() { - if (centroids == null) - run(); - return centroids; - } - - /* - * X - set of vectors compute the medoid of a vector set - */ - float[] medoid(List X) { - float[] ret = X.get(0); - for (int i = 1; i < X.size(); i++) { - for (int j = 0; j < ret.length; j++) { - ret[j] += X.get(i)[j]; - } - } - for (int j = 0; j < ret.length; j++) { - ret[j] = ret[j] / ((float) X.size()); - } - return ret; - } - - // float[] rngvec; the range vector is moot if incoming data has been - // normalized - // post normalization it should all be zero centered, with variance 1 - - /* - * super simple hash algorithm, reminiscient of pstable lsh - */ - public long hashvec(float[] xt, float[] x, - Map> IDAndCent, - Map> IDAndLabel, int ct) { - long s = 1;// fixes leading 0's bug - for (int i = 0; i < xt.length; i++) { - s <<= 1; - if (xt[i] > rngvec[i]) - s += 1; - if (IDAndCent.containsKey(s)) { - if (IDAndLabel.get(s) != null) - IDAndLabel.get(s).add(ct); - if (IDAndCent.get(s) != null) - IDAndCent.get(s).add(x); - } else { - ArrayList xlist = new ArrayList<>(); - xlist.add(x); - IDAndCent.put(s, xlist); - ArrayList idlist = new ArrayList<>(); - idlist.add(ct); - IDAndLabel.put(s, idlist); - } - } - return s; - } - - /* - * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid - * vector map - * - * hash the projected vector x and update the hash to centroid and counts - * maps - */ - void addtocounter(float[] x, Projector p, - Map> IDAndCent, - Map> IDandID, int ct) { - float[] xt = p.project(x); - - hashvec(xt, x, IDAndCent, IDandID, ct); - } - - /* - * X - data set k - canonical k in k-means l - clustering sub-space Compute - * density mode via iterative deepening hash counting - */ - public Collection> findDensityModes() - throws InterruptedException, ExecutionException { - - // #create projector matrixs - Projector projector = so.getProjectionType(); - projector.setOrigDim(so.getdim()); - projector.setProjectedDim(so.getDimparameter()); - projector.setRandomSeed(so.getRandomSeed()); - projector.init(); - - // int ct = 1; - - List dat = so.getRawData(); - - //this counter gets shared - AtomicInteger ct = new AtomicInteger(0); - - ForkJoinPool executor = new ForkJoinPool(this.threads); - - int chunksize = dat.size() / this.threads; - - //This is the array of essentially thread objets that process in parallel - ArrayList>>> gather = new ArrayList<>(this.threads); - - for (int i = 0; i < this.threads; i++) { - - int chunk = chunksize* i; - gather.add(executor.submit(new Callable>>() { - - // this is the mapper function. the dataset is split among the processing threads - // each thread performs the projections and counter adds. - // this method is sequentially bottlenecked in regard to the add part - // there are some ways to fix this, but ultimately each thread needs to maintain - // its own count-sketch. then those sketch must be merged, via the binary - // operation - public Map> call() { - Map> IDAndCent = new HashMap<>(); - Map> IDAndID = new HashMap<>(); - for (int j = chunk; j < chunksize + chunk && j < dat.size(); j++) { - addtocounter(dat.get(j), projector, IDAndCent, IDAndID, - ct.incrementAndGet()); - } - return IDAndCent ;//new Object[] { IDAndCent, IDAndID }; - } - })); - } - - List>> gatheredCent = new ArrayList<>(this.threads); -// List> gatheredID = new ArrayList<>(this.threads); - -// executor.awaitTermination(10,TimeUnit.SECONDS); - for (Future>> f : gather) { - Map> o = f.get(); - gatheredCent.add(o); -// gatheredID.add((Map) o[1]); - } - - executor.shutdown(); - - - // this function merges the centroid sets in parallel. - // it would be the basis of the reduce part - // even though the functions are called map, the return is a collection/gather operation - Map IDAndCent = gatheredCent - .stream() - .parallel() - .map(Map::entrySet) - .flatMap(Collection::stream) - .collect( - Collectors.toConcurrentMap(Map.Entry::getKey, - Map.Entry::getValue, - (old, latest)->{ - old.addAll(latest); - return old; - } - )); - - - //this is sequential... - // next we want to prune the tree by parent count comparison - // follows breadthfirst search - HashMap denseSetOfIDandCount = new HashMap(); - for (Long cur_id : new TreeSet(IDAndCent.keySet())) { - if (cur_id > so.getk()) { - int cur_count = IDAndCent.get(cur_id).size(); - long parent_id = cur_id >>> 1; - int parent_count = IDAndCent.get(parent_id).size(); - - if (cur_count != 0 && parent_count != 0) { - if (cur_count == parent_count) { - denseSetOfIDandCount.put(parent_id, 0L); - IDAndCent.put(parent_id, new ArrayList<>()); - denseSetOfIDandCount.put(cur_id, (long) cur_count); - } else { - if (2 * cur_count > parent_count) { - denseSetOfIDandCount.remove(parent_id); - IDAndCent.put(parent_id, new ArrayList<>()); - denseSetOfIDandCount.put(cur_id, (long) cur_count); - } - } - } - } - } - - // remove keys with support less than 1 - Stream> stream = denseSetOfIDandCount.entrySet() - .parallelStream().filter(p -> p.getValue() > 1); - // 64 so 6 bits? - // stream = stream.filter(p -> p.getKey() > 64); - - List sortedIDList = new ArrayList<>(); - // sort and limit the list - stream.sorted(Entry. comparingByValue().reversed()) - .limit(so.getk() * 4).parallel() - .forEachOrdered(x -> sortedIDList.add(x.getKey())); - - // compute centroids - - HashMap> estcents = new HashMap<>(); - for (int i = 0; i < sortedIDList.size(); i++) - { - estcents.put(sortedIDList.get(i), - new ArrayList(IDAndCent.get(sortedIDList.get(i)))); - } - - return estcents.values(); - } - - public void run() { - rngvec = new float[so.getDimparameter()]; - Random r = new Random(so.getRandomSeed()); - for (int i = 0; i < so.getDimparameter(); i++) - rngvec[i] = (float) r.nextGaussian(); - - Collection> clustermembers; - try { - clustermembers = findDensityModes(); - - List centroids = new ArrayList<>(); - - List weights = new ArrayList<>(); - int k = clustermembers.size() > 200 + so.getk() ? 200 + so.getk() - : clustermembers.size(); - - for (List cl : clustermembers) { - weights.add(new Float(cl.size())); - centroids.add(medoid(cl)); - } - - Agglomerative3 aggloOffline = new Agglomerative3(centroids, - so.getk()); - aggloOffline.setWeights(weights); - this.centroids = aggloOffline.getCentroids(); - } catch (InterruptedException | ExecutionException e) { - - e.printStackTrace(); - } - } - - public static void main(String[] args) throws FileNotFoundException, - IOException { - - int k = 10; - int d = 1000; - int n = 10000; - float var = 1.1f; - int count = 10; - System.out.printf("ClusterVar\t"); - for (int i = 0; i < count; i++) - System.out.printf("Trial%d\t", i); - System.out.printf("RealWCSS\n"); - - for (float f = var; f < 5.01; f += .05f) { - float avgrealwcss = 0; - float avgtime = 0; - System.out.printf("%f\t", f); - for (int i = 0; i < count; i++) { - GenerateData gen = new GenerateData(k, n / k, d, f, true, 1f); - // gen.writeCSVToFile(new - // File("/home/lee/Desktop/reclsh/in.csv")); - RPHashObject o = new SimpleArrayReader(gen.data, k); - o.setDimparameter(32); - - RPHashAdaptive2PassParallel rphit = new RPHashAdaptive2PassParallel( - o); - long startTime = System.nanoTime(); - List centsr = rphit.getCentroids(); - - avgtime += (System.nanoTime() - startTime) / 100000000; - - avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), - gen.getData()); - - System.out.printf("%.0f\t", - StatTests.WCSSECentroidsFloat(centsr, gen.data)); - System.gc(); - } - System.out.printf("%.0f\n", avgrealwcss / count); - } - } - - @Override - public RPHashObject getParam() { - return so; - } - - @Override - public void setWeights(List counts) { - // TODO Auto-generated method stub - - } - - @Override - public void setData(List centroids) { - this.centroids = centroids; - - } - - @Override - public void setRawData(List centroids) { - if (this.centroids == null) - this.centroids = new ArrayList<>(centroids.size()); - for (float[] f : centroids) { - this.centroids.add(new Centroid(f, 0)); - } - } - - @Override - public void setK(int getk) { - this.so.setK(getk); - } - - @Override - public void reset(int randomseed) { - centroids = null; - so.setRandomSeed(randomseed); - } - - @Override - public boolean setMultiRun(int runs) { - return false; - } -} diff --git a/src/main/java/edu/uc/rphash/RPHashMultiProj.java b/src/main/java/edu/uc/rphash/RPHashMultiProj.java deleted file mode 100644 index 18f6f47..0000000 --- a/src/main/java/edu/uc/rphash/RPHashMultiProj.java +++ /dev/null @@ -1,307 +0,0 @@ -package edu.uc.rphash; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Random; - -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.Readers.SimpleArrayReader; -import edu.uc.rphash.decoders.Decoder; -import edu.uc.rphash.decoders.Leech; -import edu.uc.rphash.decoders.Spherical; -import edu.uc.rphash.frequentItemSet.ItemSet; -import edu.uc.rphash.frequentItemSet.SimpleFrequentItemSet; -import edu.uc.rphash.lsh.LSH; -import edu.uc.rphash.projections.Projector; -import edu.uc.rphash.standardhash.HashAlgorithm; -import edu.uc.rphash.standardhash.NoHash; -import edu.uc.rphash.tests.StatTests; -import edu.uc.rphash.tests.generators.GenerateData; - -/** - * This is the correlated multi projections approach. In this RPHash variation - * we try to incorporate the advantage of multiple random projections in order - * to combat increasing cluster error rates as the deviation between projected - * and full data increases. The main idea is similar to the referential RPHash, - * however the set union is projection id dependent. This will be done in a - * simplified bitmask addition to the hash code in lieu of an array of sets data - * structures. - * - * @author lee - * - */ -public class RPHashMultiProj implements Clusterer { - float variance; - - public RPHashObject map() { - Iterator vecs = so.getVectorIterator(); - if (!vecs.hasNext()) - return so; - - long[] hash; - int projections = so.getNumProjections(); - - int k = (int) (so.getk() * 2); - - // initialize our counter - ItemSet is = new SimpleFrequentItemSet(k); - // create our LSH Device - // create same LSH Device as before - - Random r = new Random(so.getRandomSeed()); - LSH[] lshfuncs = new LSH[projections]; - Decoder dec = so.getDecoderType(); - dec.setCounter(is); - HashAlgorithm hal = new NoHash(so.getHashmod()); - - // create same projection matrices as before - for (int i = 0; i < projections; i++) { - Projector p = so.getProjectionType(); - p.setOrigDim(so.getdim()); - p.setProjectedDim(dec.getDimensionality()); - p.setRandomSeed(r.nextLong()); - p.init(); - - List noise = LSH.genNoiseTable(dec.getDimensionality(), - so.getNumBlur(), r, - dec.getErrorRadius() / dec.getDimensionality()); - - lshfuncs[i] = new LSH(dec, p, hal, noise, so.getNormalize()); - } - - // add to frequent itemset the hashed Decoded randomly projected vector - while (vecs.hasNext()) { - float[] vec = vecs.next(); - // iterate over the multiple projections - for (LSH lshfunc : lshfuncs) { - // could do a big parallel projection here - hash = lshfunc.lshHashRadius(vec, so.getNumBlur()); - for (long hh : hash) { - is.add(hh); - } - } - } - so.setPreviousTopID(is.getTop()); - List countsAsFloats = new ArrayList(); - for (long ct : is.getCounts()) - countsAsFloats.add((float) ct); - so.setCounts(countsAsFloats); - return so; - } - - /* - * This is the second phase after the top ids have been in the reduce phase - * aggregated - */ - public RPHashObject reduce() { - Iterator vecs = so.getVectorIterator(); - if (!vecs.hasNext()) - return so; - - // make a set of k default centroid objects - ArrayList centroids = new ArrayList(); - for (long id : so.getPreviousTopID()) - centroids.add(new Centroid(so.getdim(), id, -1)); - - long[] hash; - int projections = so.getNumProjections(); - - // create our LSH Device - // create same LSH Device as before - Random r = new Random(so.getRandomSeed()); - LSH[] lshfuncs = new LSH[projections]; - Decoder dec = so.getDecoderType(); - HashAlgorithm hal = new NoHash(so.getHashmod()); - - // create same projection matrices as before - for (int i = 0; i < projections; i++) { - Projector p = so.getProjectionType(); - p.setOrigDim(so.getdim()); - p.setProjectedDim(dec.getDimensionality()); - p.setRandomSeed(r.nextLong()); - p.init(); - List noise = LSH.genNoiseTable(dec.getDimensionality(), - so.getNumBlur(), r, - dec.getErrorRadius() / dec.getDimensionality()); - lshfuncs[i] = new LSH(dec, p, hal, noise, so.getNormalize()); - } - - while (vecs.hasNext()) { - float[] vec = vecs.next(); - // iterate over the multiple projections - for (LSH lshfunc : lshfuncs) { - // could do a big parallel projection here - hash = lshfunc.lshHashRadius(vec, so.getNumBlur()); - for (Centroid cent : centroids) { - for (long hh : hash) { - if (cent.ids.contains(hh)) { - cent.updateVec(vec); - cent.addID(hh); - } - } - } - } - } - so.setCentroids(centroids); - return so; - } - - private List centroids = null; - private RPHashObject so; - private int runs; - - public RPHashMultiProj(int k, List data) { - so = new SimpleArrayReader(data, k); - runs = 1; - } - - public RPHashMultiProj(RPHashObject so) { - this.so = so; - } - - public RPHashMultiProj() { - so = new SimpleArrayReader(); - } - - public List getCentroids(RPHashObject so) { - this.so = so; - - if (centroids == null) - run(); - return centroids; - } - - @Override - public List getCentroids() { - if (centroids == null) { - run(); - } - return centroids; - } - - private void run() { - runs = 1; - double minwcss = Double.MAX_VALUE; - List mincentroids = new ArrayList<>(); - for (int currun = 0; currun < runs;) { - - map(); - reduce(); - - Clusterer offlineclusterer = so.getOfflineClusterer(); - List tmpcents; - if (offlineclusterer != null) { - offlineclusterer.setMultiRun(1);// is deterministic - offlineclusterer.setData(so.getCentroids()); - offlineclusterer.setWeights(so.getCounts()); - offlineclusterer.setK(so.getk()); - tmpcents = offlineclusterer.getCentroids(); - } else { - tmpcents = so.getCentroids().subList(0, so.getk()); - } - - if (tmpcents.size() == so.getk()) {// skip bad clusterings - double tmpwcss = StatTests.WCSSECentroidsFloat(tmpcents, - so.getRawData()); - // System.out.println(tmpwcss + ":" + so.getCounts()); - if (tmpwcss < minwcss) { - minwcss = tmpwcss; - mincentroids = tmpcents; - } - currun++; - } - - this.reset(new Random().nextInt()); - - } - - this.centroids = mincentroids; - } - - public static void main(String[] args) { - - int k = 10; - int d = 1000; - int n = 10000; - float var = .6f; - int count = 5; - System.out.printf("Decoder: %s\n","Spherical"); - System.out.printf("ClusterVar\t"); - for (int i = 0; i < count; i++) - System.out.printf("Trial%d\t", i); - System.out.printf("RealWCSS\n"); - - - - for (float f = var; f < 3.01; f += .1f) { - float avgrealwcss = 0; - float avgtime = 0; - System.out.printf("%f\t", f); - for (int i = 0; i < count; i++) { - GenerateData gen = new GenerateData(k, n / k, d, f, true, 1f); - RPHashObject o = new SimpleArrayReader(gen.data, k); - o.setDecoderType(new Spherical(32,4,1)); - o.setDimparameter(32); - RPHashMultiProj rphit = new RPHashMultiProj(o); - long startTime = System.nanoTime(); - List centsr = rphit.getCentroids(); - - avgtime += (System.nanoTime() - startTime) / 100000000; - - avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), - gen.getData()); - - System.out.printf("%.0f\t", - StatTests.WCSSECentroidsFloat(centsr, gen.data)); - System.gc(); - - } - System.out.printf("%.0f\n", avgrealwcss / count); - } - } - - @Override - public RPHashObject getParam() { - return so; - } - - @Override - public void setWeights(List counts) { - } - - @Override - public void setData(List data) { - centroids = new ArrayList<>(); - for (Centroid c : data) { - so.addRawData(c.centroid); - } - so.setDimparameter(data.get(0).dimensions); - } - - @Override - public void setK(int getk) { - this.so.setK(getk); - } - - @Override - public void setRawData(List data) { - so.setRawData(data); - this.so.setDimparameter(data.get(0).length); - } - - @Override - public void reset(int randomseed) { - centroids = null; - so.setRandomSeed(randomseed); - } - - @Override - public boolean setMultiRun(int runs) { - this.runs = runs; - return true; - } - -} diff --git a/src/main/java/edu/uc/rphash/RPHashStreamingAK.java b/src/main/java/edu/uc/rphash/RPHashStreamingAK.java deleted file mode 100644 index 873c20d..0000000 --- a/src/main/java/edu/uc/rphash/RPHashStreamingAK.java +++ /dev/null @@ -1,198 +0,0 @@ -package edu.uc.rphash; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Random; - -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.Readers.SimpleArrayReader; -import edu.uc.rphash.decoders.Decoder; -import edu.uc.rphash.frequentItemSet.KHHCentroidCounterPush; -import edu.uc.rphash.knee.LpointKnee; -import edu.uc.rphash.lsh.LSH; -import edu.uc.rphash.projections.DBFriendlyProjection; -import edu.uc.rphash.projections.Projector; -import edu.uc.rphash.standardhash.HashAlgorithm; -import edu.uc.rphash.standardhash.MurmurHash; -import edu.uc.rphash.tests.StatTests; -import edu.uc.rphash.tests.clusterers.LloydIterativeKmeans; -import edu.uc.rphash.tests.generators.ClusterGenerator; - -/**This is an adaptation of RPHash Streaming with support for - * automatic knee finding and time based cluster decay. - * @author lee - * - */ -public class RPHashStreamingAK implements StreamClusterer { - - public KHHCentroidCounterPush is; - private LSH[] lshfuncs; - private StatTests vartracker; - private List centroids = null; - private RPHashObject so; - - - @Override - public synchronized long addVectorOnlineStep(float[] vec) { - - if(!lshfuncs[0].lshDecoder.selfScaling()){ - this.vartracker.updateVarianceSampleVec(vec); - vec = this.vartracker.scaleVector(vec); - } - - - Centroid c = new Centroid(vec,-1); - int ret = -1; - - for (LSH lshfunc : lshfuncs) { - if (so.getNumBlur() != 1) { - long[] hash = lshfunc - .lshHashRadius(vec, so.getNumBlur()); - for (long h : hash) { - c.addID(h); - is.addLong(h, 1); - } - } else { - long hash = lshfunc.lshHash(vec); - c.addID(hash); - is.addLong(hash, 1); - } - } - ret = is.addAndUpdate(c); - - return ret; - } - - public void init() { - Random r = new Random(so.getRandomSeed()); - this.vartracker = new StatTests(.01f); - int projections = so.getNumProjections(); - - // initialize our counter - float decayrate = so.getDecayRate();// 1f;// bottom number is window - // size - is = new KHHCentroidCounterPush(decayrate,new LpointKnee()); - // create LSH Device - lshfuncs = new LSH[projections]; - Decoder dec = so.getDecoderType(); - HashAlgorithm hal = new MurmurHash(so.getHashmod()); - // create projection matrices add to LSH Device - for (int i = 0; i < projections; i++) { - Projector p = new DBFriendlyProjection(so.getdim(), - dec.getDimensionality(), r.nextLong()); - List noise = LSH.genNoiseTable(dec.getDimensionality(), - so.getNumBlur(), r, dec.getErrorRadius() - / dec.getDimensionality()); - lshfuncs[i] = new LSH(dec, p, hal, noise,so.getNormalize()); - } - } - - public RPHashStreamingAK(ClusterGenerator c) { - so = new SimpleArrayReader(c,0); - init(); - } - - public RPHashStreamingAK(RPHashObject so) { - this.so = so; - init(); - } - - - - @Override - public List getCentroids() { - if (centroids == null) { - init(); - run(); - getCentroidsOfflineStep(); - } - return centroids; - } - - public List getCentroidsOfflineStep() { - - centroids = is.getTop(); - - -// centroids = new ArrayList(); -// List counts = is.getCounts(); -// -// for (int i = 0; i < cents.size(); i++) { -// centroids.add(cents.get(i).centroid()); -// } - - Clusterer offlineclusterer = so.getOfflineClusterer(); - offlineclusterer.setWeights(so.getCounts()); - offlineclusterer.setData(so.getCentroids()); - offlineclusterer.setK(so.getk()); - centroids = offlineclusterer.getCentroids(); - - return centroids; - } - - public void run() { - // add to frequent itemset the hashed Decoded randomly projected - // vector - Iterator vecs = so.getVectorIterator(); - while (vecs.hasNext()) { - addVectorOnlineStep(vecs.next()); - } - } - - public List getTopIdSizes() { - return is.getCounts(); - } - - @Override - public RPHashObject getParam() { - return this.so; - } - - @Override - public void setWeights(List counts) { - // TODO Auto-generated method stub - - } - - @Override - public void setRawData(List data) { -// this.data = data; - } - - @Override - public void setData(List centroids) { - ArrayList data = new ArrayList(centroids.size()); - for(Centroid c : centroids)data.add(c.centroid()); - setRawData(data); - } - - @Override - public void setK(int getk) { - // TODO Auto-generated method stub - - } - - @Override - public void shutdown() { - // TODO Auto-generated method stub - - } - - @Override - public void reset(int randomseed) { - centroids = null; - so.setRandomSeed(randomseed); - } - - @Override - public boolean setMultiRun(int runs) { - return false; - } - - @Override - public int getProcessors() { - return 1; - } - -} diff --git a/src/main/java/edu/uc/rphash/Readers/RPHashObject.java b/src/main/java/edu/uc/rphash/Readers/RPHashObject.java index 499ae9f..9f38946 100644 --- a/src/main/java/edu/uc/rphash/Readers/RPHashObject.java +++ b/src/main/java/edu/uc/rphash/Readers/RPHashObject.java @@ -8,20 +8,21 @@ import edu.uc.rphash.decoders.Decoder; import edu.uc.rphash.decoders.DepthProbingLSH; import edu.uc.rphash.decoders.E8; -import edu.uc.rphash.decoders.Golay; + import edu.uc.rphash.decoders.Leech; import edu.uc.rphash.decoders.MultiDecoder; -import edu.uc.rphash.decoders.PsdLSH; + import edu.uc.rphash.decoders.Spherical; import edu.uc.rphash.projections.DBFriendlyProjection; +import edu.uc.rphash.projections.GaussianProjection; import edu.uc.rphash.projections.Projector; -import edu.uc.rphash.tests.clusterers.Agglomerative; + import edu.uc.rphash.tests.clusterers.Agglomerative3; import edu.uc.rphash.tests.clusterers.Agglomerative3.ClusteringType; import edu.uc.rphash.tests.clusterers.KMeans2; import edu.uc.rphash.tests.clusterers.KMeans2NoWCSS; import edu.uc.rphash.tests.clusterers.KMeansPlusPlus; -import edu.uc.rphash.tests.clusterers.Kmeans; + import edu.uc.rphash.tests.clusterers.MultiKMPP; import edu.uc.rphash.tests.clusterers.DBScan; @@ -41,8 +42,9 @@ public interface RPHashObject { //final static Clusterer DEFAULT_OFFLINE_CLUSTERER = new MultiKMPP(); - final static Projector DEFAULT_PROJECTOR = new DBFriendlyProjection(); - + final static Projector DEFAULT_PROJECTOR = new DBFriendlyProjection(); + //final static Projector DEFAULT_PROJECTOR = new GaussianProjection(); + int getdim(); Iterator getVectorIterator(); diff --git a/src/main/java/edu/uc/rphash/TWRP1.java b/src/main/java/edu/uc/rphash/TWRP1.java new file mode 100644 index 0000000..aaf3540 --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRP1.java @@ -0,0 +1,763 @@ +package edu.uc.rphash; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; +import java.util.Map; + + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; + + + +public class TWRP1 implements Clusterer, Runnable { + + boolean znorm = false; + + + private int counter; + private float[] rngvec; + private List centroids = null; + + + private RPHashObject so; + + public TWRP1(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run2(); + return centroids; + } + + + + + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + + //float[] rngvec; the range vector is moot if incoming data has been normalized + //post normalization it should all be zero centered, with variance 1 + + /* + * super simple hash algorithm, reminiscient of pstable lsh + */ + // xt is the projected vector and x is the original vector , rngvec is the randomly generated vector of projected dim. + + public long hashvec(float[] xt, float[] x, + HashMap> IDAndCent, HashMap> IDAndLabel,int ct) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) +// s += 1; + s= s+1; + + if (IDAndCent.containsKey(s)) { + IDAndLabel.get(s).add(ct); + IDAndCent.get(s).add(x); + } else { + List xlist = new ArrayList<>(); + xlist.add(x); + IDAndCent.put(s, xlist); + List idlist = new ArrayList<>(); + idlist.add(ct); + IDAndLabel.put(s, idlist); + } + } + return s; + } + + + /* + * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid + * vector map + * + * hash the projected vector x and update the hash to centroid and counts + * maps + */ + void addtocounter(float[] x, Projector p, + HashMap> IDAndCent,HashMap> IDandID,int ct) { + float[] xt = p.project(x); + +// counter++; +// for(int i = 0;i> IDAndCent,HashMap> IDandID,int ct,float[] mean,float[] variance) + { + float[] xt = p.project(StatTests.znormvec(x, mean, variance)); + +// counter++; +// for(int i = 0;i> findDensityModes() { + HashMap> IDAndCent = new HashMap<>(); + HashMap> IDAndID = new HashMap<>(); + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + + int ct = 0; +// if(znorm == true){ +// float[] variance = StatTests.varianceCol(so.getRawData()); +// float[] mean = StatTests.meanCols(so.getRawData()); +// // #process data by adding to the counter +// for (float[] x : so.getRawData()) +// { +// addtocounter(x, projector, IDAndCent,IDAndID,ct++,mean,variance); +// } +// } +// +// else + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, IDAndCent, IDAndID,ct++); + } + } + + + for (Long name: IDAndCent.keySet()){ + + String key =name.toString(); + System.out.println(key ); + + // String value = IDAndCent.get(name).toString() ; + // String value1 = Arrays.toString(value.toString()); + + // System.out.println(key + " " + value); + + +} + + for (Long name: IDAndID.keySet()){ + + // String key =name.toString(); + // String value = IDAndID.get(name).toString(); + // System.out.println(key + " " + value); + + +} + + // we would compress the hashmaps. SetOfIDandCount has the ids and the counts corresponding to that id. + // we have two hashmaps: 1. IDAndCent and 2. IDAndID. we will use IDAndCent + + + HashMap MapOfIDAndCount = new HashMap(); + + HashMap MapOfIDAndCent = new HashMap(); + + for (Long cur_id : new TreeSet(IDAndCent.keySet())) + { + int cur_count = IDAndCent.get(cur_id).size(); + + MapOfIDAndCount.put(cur_id, (long) cur_count); // this has the hashids and counts. + + List bucketpoints = new ArrayList<>(); + + Iterator e = IDAndCent.get(cur_id).iterator(); + + // int i=1; + while (e.hasNext()) { + + // System.out.println(i++); + + bucketpoints.add(e.next()) ; + + } + + float [] bucketcent; + + bucketcent = medoid(bucketpoints); + + MapOfIDAndCent.put(cur_id, bucketcent); // this has the hashids and centroids. + + // System.out.println(cur_id + " " + cur_count); + + // int c = MapOfIDAndCent.get(cur_id).length; + + // System.out.println(cur_id + " " + c); + + + } + + + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + HashMap denseSetOfIDandCount = new HashMap(); + for (Long cur_id : new TreeSet(IDAndCent.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = IDAndCent.get(cur_id).size(); + long parent_id = cur_id>>>1; + int parent_count = IDAndCent.get(parent_id).size(); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount.put(parent_id, 0L); + IDAndCent.put(parent_id, new ArrayList<>()); + denseSetOfIDandCount.put(cur_id, (long) cur_count); + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount.remove(parent_id); + IDAndCent.put(parent_id, new ArrayList<>()); + denseSetOfIDandCount.put(cur_id, (long) cur_count); + } + } + } + } + } + + + + HashMap denseSetOfIDandCount2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + + //HashMap> IDAndCent = new HashMap<>(); and HashMap MapOfIDAndCent = new HashMap(); + + MapOfIDAndCent.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + + + //remove keys with support less than 1 + Stream> stream = denseSetOfIDandCount.entrySet().stream().filter(p -> p.getValue() > 1); + + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + //64 so 6 bits? + //stream = stream.filter(p -> p.getKey() > 64); + + List sortedIDList= new ArrayList<>(); + // sort and limit the list + stream.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*4) + .forEachOrdered(x -> sortedIDList.add(x.getKey())); + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*4) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + + + + // compute centroids + + HashMap> estcents = new HashMap<>(); + for (int i =0; i KeyAndCent = new HashMap<>(); + HashMap KeyAndCount = new HashMap<>(); + HashMap WeightAndCent = new HashMap<>(); + + for (int i =0; i(estcents.values()); + } + + + + + + + + + + + + public HashMap findDensityModes2() { + HashMap> IDAndCent = new HashMap<>(); + HashMap> IDAndID = new HashMap<>(); + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + + int ct = 0; +// if(znorm == true){ +// float[] variance = StatTests.varianceCol(so.getRawData()); +// float[] mean = StatTests.meanCols(so.getRawData()); +// // #process data by adding to the counter +// for (float[] x : so.getRawData()) +// { +// addtocounter(x, projector, IDAndCent,IDAndID,ct++,mean,variance); +// } +// } +// +// else + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, IDAndCent, IDAndID,ct++); + } + } + + + for (Long name: IDAndCent.keySet()){ + + String key =name.toString(); + System.out.println(key ); + + // String value = IDAndCent.get(name).toString() ; +// String value1 = Arrays.toString(value.toString()); + +// System.out.println(key + " " + value); + + +} + + for (Long name: IDAndID.keySet()){ + +// String key =name.toString(); +// String value = IDAndID.get(name).toString(); +// System.out.println(key + " " + value); + + +} + + // we would compress the hashmaps. SetOfIDandCount has the ids and the counts corresponding to that id. + // we have two hashmaps: 1. IDAndCent and 2. IDAndID. we will use IDAndCent + + + HashMap MapOfIDAndCount = new HashMap(); + + HashMap MapOfIDAndCent = new HashMap(); + + for (Long cur_id : new TreeSet(IDAndCent.keySet())) + { + int cur_count = IDAndCent.get(cur_id).size(); + + MapOfIDAndCount.put(cur_id, (long) cur_count); // this has the hashids and counts. + + List bucketpoints = new ArrayList<>(); + + Iterator e = IDAndCent.get(cur_id).iterator(); + +// int i=1; + while (e.hasNext()) { + +// System.out.println(i++); + + bucketpoints.add(e.next()) ; + + } + + float [] bucketcent; + + bucketcent = medoid(bucketpoints); + + MapOfIDAndCent.put(cur_id, bucketcent); // this has the hashids and centroids. + +// System.out.println(cur_id + " " + cur_count); + + // int c = MapOfIDAndCent.get(cur_id).length; + + // System.out.println(cur_id + " " + c); + + + } + + + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + HashMap denseSetOfIDandCount = new HashMap(); + for (Long cur_id : new TreeSet(IDAndCent.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = IDAndCent.get(cur_id).size(); + long parent_id = cur_id>>>1; + int parent_count = IDAndCent.get(parent_id).size(); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount.put(parent_id, 0L); + IDAndCent.put(parent_id, new ArrayList<>()); + denseSetOfIDandCount.put(cur_id, (long) cur_count); + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount.remove(parent_id); + IDAndCent.put(parent_id, new ArrayList<>()); + denseSetOfIDandCount.put(cur_id, (long) cur_count); + } + } + } + } + } + + + + HashMap denseSetOfIDandCount2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + +//HashMap> IDAndCent = new HashMap<>(); and HashMap MapOfIDAndCent = new HashMap(); + + MapOfIDAndCent.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + + + //remove keys with support less than 1 + Stream> stream = denseSetOfIDandCount.entrySet().stream().filter(p -> p.getValue() > 1); + + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + //64 so 6 bits? + //stream = stream.filter(p -> p.getKey() > 64); + + List sortedIDList= new ArrayList<>(); + // sort and limit the list + stream.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*4) + .forEachOrdered(x -> sortedIDList.add(x.getKey())); + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*4) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + + + + // compute centroids + + HashMap> estcents = new HashMap<>(); + for (int i =0; i KeyAndCent = new HashMap<>(); + HashMap KeyAndCount = new HashMap<>(); + HashMap WeightAndCent = new HashMap<>(); + + for (int i =0; i(estcents.values()); + + return WeightAndCent; + + +} + + + + + + + public void run() { + rngvec = new float[so.getDimparameter()]; + counter = 0; + Random r = new Random(so.getRandomSeed()); + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) r.nextGaussian(); + + List> clustermembers = findDensityModes(); + Listcentroids = new ArrayList<>(); + + List weights =new ArrayList<>(); + int k = clustermembers.size()>200+so.getk()?200+so.getk():clustermembers.size(); + for(int i=0;i WeightAndClusters = findDensityModes2(); + + + Listcentroids2 = new ArrayList<>(); + List weights2 =new ArrayList<>(); + + + int NumberOfMicroClusters = WeightAndClusters.size() ; + + + int k = NumberOfMicroClusters>200+so.getk()?200+so.getk():NumberOfMicroClusters; + + + // have to prune depending NumberOfMicroClusters returned. + + for (Long weights : new TreeSet(WeightAndClusters.keySet())) + + { + weights2.add((float)weights); + centroids2.add(WeightAndClusters.get(weights)); + } + + + + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); + aggloOffline.setWeights(weights2); + + this.centroids = aggloOffline.getCentroids(); + + + } + + + + + + public static void main(String[] args) throws FileNotFoundException, + IOException { + + int k = 6;//6; + int d = 64;//16; + int n = 700; + float var = .5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + for (float f = var; f < 1.01; f += 1.5f) { + float avgrealwcss = 0; + float avgtime = 0; + // System.out.printf("%f\t", f); + for (int i = 0; i < count; i++) { + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); + // gen.writeCSVToFile(new + // File("/home/lee/Desktop/reclsh/in.csv")); + RPHashObject o = new SimpleArrayReader(gen.data, k); + o.setDimparameter(4); + TWRP1 rphit = new TWRP1(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + // System.out.printf("%.0f\t", + // StatTests.WCSSECentroidsFloat(centsr, gen.data)); + // System.gc(); + } + System.out.printf("%.0f\n", avgrealwcss / count); + + } + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } +} diff --git a/src/main/java/edu/uc/rphash/TWRPv2.java b/src/main/java/edu/uc/rphash/TWRPv2.java new file mode 100644 index 0000000..08b05a2 --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRPv2.java @@ -0,0 +1,534 @@ +package edu.uc.rphash; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; +import java.util.Map; + + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; + + + +public class TWRPv2 implements Clusterer, Runnable { + + boolean znorm = false; + + + private int counter; + private float[] rngvec; + private List centroids = null; + + + private RPHashObject so; + + public TWRPv2(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + return centroids; + } + + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + + + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, + float cnt_2, float[] x_2) { + + float cnt_r = cnt_1 + cnt_2; + + float[] x_r = new float[x_1.length]; + + for (int i = 0; i < x_1.length; i++) { + x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; + + } + + float[][] ret = new float[2][]; + ret[0] = new float[1]; + ret[0][0] = cnt_r; + ret[1] = x_r; + return ret; + } + + + + + + //float[] rngvec; the range vector is moot if incoming data has been normalized + //post normalization it should all be zero centered, with variance 1 + + /* + * super simple hash algorithm, reminiscient of pstable lsh + */ + // xt is the projected vector and x is the original vector , rngvec is the randomly generated vector of projected dim. + + public long hashvec(float[] xt, float[] x, + HashMap> IDAndCent, HashMap> IDAndLabel,int ct) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) +// s += 1; + s= s+1; + + if (IDAndCent.containsKey(s)) { + IDAndLabel.get(s).add(ct); + IDAndCent.get(s).add(x); + } else { + List xlist = new ArrayList<>(); + xlist.add(x); + IDAndCent.put(s, xlist); + List idlist = new ArrayList<>(); + idlist.add(ct); + IDAndLabel.put(s, idlist); + } + } + return s; + } + + public long hashvec2(float[] xt, float[] x, + HashMap MapOfIDAndCent, HashMap MapOfIDAndCount,int ct) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) +// s += 1; + s= s+1; + + + + if (MapOfIDAndCent.containsKey(s)) { + + float CurrentCount = MapOfIDAndCount.get(s); + float CurrentCent [] = MapOfIDAndCent.get(s); + float CountForIncomingVector = 1; + float IncomingVector [] = x; + + float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, CountForIncomingVector, IncomingVector ); + + Long UpdatedCount = (long) MergedValues[0][0] ; + + float[] MergedVector = MergedValues[1] ; + + MapOfIDAndCount.put(s , UpdatedCount); + + MapOfIDAndCent.put(s, MergedVector); + + } + + + else { + + float[] xlist = x; + MapOfIDAndCent.put(s, xlist); + MapOfIDAndCount.put(s, (long)1); + } + } + return s; + } + + + /* + * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid + * vector map + * + * hash the projected vector x and update the hash to centroid and counts + * maps + */ + void addtocounter(float[] x, Projector p, + HashMap> IDAndCent,HashMap> IDandID,int ct) { + float[] xt = p.project(x); + +// counter++; +// for(int i = 0;i> IDAndCent,HashMap> IDandID,int ct,float[] mean,float[] variance) + { + float[] xt = p.project(StatTests.znormvec(x, mean, variance)); + +// counter++; +// for(int i = 0;i findDensityModes2() { + HashMap> IDAndCent = new HashMap<>(); + HashMap> IDAndID = new HashMap<>(); + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + + int ct = 0; +// if(znorm == true){ +// float[] variance = StatTests.varianceCol(so.getRawData()); +// float[] mean = StatTests.meanCols(so.getRawData()); +// // #process data by adding to the counter +// for (float[] x : so.getRawData()) +// { +// addtocounter(x, projector, IDAndCent,IDAndID,ct++,mean,variance); +// } +// } +// +// else + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, IDAndCent, IDAndID,ct++); + } + } + + + for (Long name: IDAndCent.keySet()){ + + String key =name.toString(); + System.out.println(key ); + + // String value = IDAndCent.get(name).toString() ; +// String value1 = Arrays.toString(value.toString()); + +// System.out.println(key + " " + value); + + +} + + for (Long name: IDAndID.keySet()){ + +// String key =name.toString(); +// String value = IDAndID.get(name).toString(); +// System.out.println(key + " " + value); + + +} + + // we would compress the hashmaps. SetOfIDandCount has the ids and the counts corresponding to that id. + // we have two hashmaps: 1. IDAndCent and 2. IDAndID. we will use IDAndCent + + + HashMap MapOfIDAndCount = new HashMap(); + + HashMap MapOfIDAndCent = new HashMap(); + + for (Long cur_id : new TreeSet(IDAndCent.keySet())) + { + int cur_count = IDAndCent.get(cur_id).size(); + + MapOfIDAndCount.put(cur_id, (long) cur_count); // this has the hashids and counts. + + List bucketpoints = new ArrayList<>(); + + Iterator e = IDAndCent.get(cur_id).iterator(); + +// int i=1; + while (e.hasNext()) { + +// System.out.println(i++); + + bucketpoints.add(e.next()) ; + + } + + float [] bucketcent; + + bucketcent = medoid(bucketpoints); + + MapOfIDAndCent.put(cur_id, bucketcent); // this has the hashids and centroids. + +// System.out.println(cur_id + " " + cur_count); + + // int c = MapOfIDAndCent.get(cur_id).length; + + // System.out.println(cur_id + " " + c); + + + } + +// int NumberOfMicroClustersBeforePruning = MapOfIDAndCent.size() ; +// System.out.println("NumberOfMicroClustersBeforePruning = "+ NumberOfMicroClustersBeforePruning); + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + + + + HashMap denseSetOfIDandCount2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + +//HashMap> IDAndCent = new HashMap<>(); and HashMap MapOfIDAndCent = new HashMap(); + + MapOfIDAndCent.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + + + //remove keys with support less than 1 + + + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + //64 so 6 bits? + //stream = stream.filter(p -> p.getKey() > 64); + + + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*4) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + + + + + HashMap KeyAndCent = new HashMap<>(); + HashMap KeyAndCount = new HashMap<>(); + HashMap WeightAndCent = new HashMap<>(); + + for (int i =0; i WeightAndClusters = findDensityModes2(); + + + Listcentroids2 = new ArrayList<>(); + List weights2 =new ArrayList<>(); + + + int NumberOfMicroClusters = WeightAndClusters.size() ; + System.out.println("NumberOfMicroClusters = "+ NumberOfMicroClusters); + + // int k = NumberOfMicroClusters>200+so.getk()?200+so.getk():NumberOfMicroClusters; + + // have to prune depending NumberOfMicroClusters returned. + + for (Long weights : new TreeSet(WeightAndClusters.keySet())) + + { + weights2.add((float)weights); + centroids2.add(WeightAndClusters.get(weights)); + } + + + //System.out.printf("\tvalueofK is "); + //System.out.println( so.getk()); + + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); + + //System.out.println(centroids2.size()); + aggloOffline.setWeights(weights2); + + this.centroids = aggloOffline.getCentroids(); + + + } + + + + + + public static void main(String[] args) throws FileNotFoundException, + IOException { + + int k = 5;//6; + int d = 100;//16; + int n = 5000; + float var = 1.5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + String Output = "/C:/Users/user/Desktop/temp/OutputTwrpCents" ; + + float f = var; + float avgrealwcss = 0; + float avgtime = 0; + // System.out.printf("%f\t", f); + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); + // gen.writeCSVToFile(new + // File("/home/lee/Desktop/reclsh/in.csv")); + RPHashObject o = new SimpleArrayReader(gen.data, k); + o.setDimparameter(8); + + TWRPv2 rphit = new TWRPv2(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + // System.out.printf("%.0f\t", + // StatTests.WCSSECentroidsFloat(centsr, gen.data)); + // System.gc(); + + // System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } +} diff --git a/src/main/java/edu/uc/rphash/decoders/Golay.java b/src/main/java/edu/uc/rphash/decoders/Golay.java deleted file mode 100644 index 6392790..0000000 --- a/src/main/java/edu/uc/rphash/decoders/Golay.java +++ /dev/null @@ -1,350 +0,0 @@ -package edu.uc.rphash.decoders; - -import java.util.Arrays; -import java.util.Random; - -import edu.uc.rphash.frequentItemSet.Countable; -import edu.uc.rphash.standardhash.MurmurHash; -import edu.uc.rphash.util.VectorUtil; - -public class Golay implements Decoder{ - /** - * Utility methods that converts a binary string into and int. - * - * @param str a string containing a binary number - * - * @return the numeric value of the supplied string - */ - - private static int fromBinary(final String str) { - return Integer.parseInt(str, 2); - } - - /** - * Mask that preserves the last 12 bits (bits in dataword). - */ - - private static final int MASK = 0xfff; //== fromBinary("111111111111"); - - /** - * Generator matrix for the code, multiplied with a dataword to generate a codeword. - */ - - private static final int[] sGenerator = { - - fromBinary("100000000000"), - fromBinary("010000000000"), - fromBinary("001000000000"), - fromBinary("000100000000"), - fromBinary("000010000000"), - fromBinary("000001000000"), - fromBinary("000000100000"), - fromBinary("000000010000"), - fromBinary("000000001000"), - fromBinary("000000000100"), - fromBinary("000000000010"), - fromBinary("000000000001"), - - /* ALTERNATIVE MATRIX - UNUSED - fromBinary("110111000101"), - fromBinary("101110001011"), - fromBinary("011100010111"), - fromBinary("111000101101"), - fromBinary("110001011011"), - fromBinary("100010110111"), - fromBinary("000101101111"), - fromBinary("001011011101"), - fromBinary("010110111001"), - fromBinary("101101110001"), - fromBinary("011011100011"), - fromBinary("111111111110"), - */ - - fromBinary("011111111111"), - fromBinary("111011100010"), - fromBinary("110111000101"), - fromBinary("101110001011"), - fromBinary("111100010110"), - fromBinary("111000101101"), - fromBinary("110001011011"), - fromBinary("100010110111"), - fromBinary("100101101110"), - fromBinary("101011011100"), - fromBinary("110110111000"), - fromBinary("101101110001"), - }; - - /** - * Transpose of the generator matrix, multiplied with a codeword to generate a syndrome. - */ - - private static final int[] sCheck = { - - fromBinary("011111111111100000000000"), - fromBinary("111011100010010000000000"), - fromBinary("110111000101001000000000"), - fromBinary("101110001011000100000000"), - fromBinary("111100010110000010000000"), - fromBinary("111000101101000001000000"), - fromBinary("110001011011000000100000"), - fromBinary("100010110111000000010000"), - fromBinary("100101101110000000001000"), - fromBinary("101011011100000000000100"), - fromBinary("110110111000000000000010"), - fromBinary("101101110001000000000001"), - - }; - - /** - * A 4096 (2^12) element array that maps datawords to codewords. - */ - - private static final int[] sCodewords; - - /** - * A 4096 (2^12) element array that maps syndromes to error bits. - */ - - private static final int[] sErrors; - - //static initialization - static { - sCodewords = computeCodewords(); - sErrors = computeErrors(); - } - - /** - * Generates the codewords array. - * - * @return an array for assignment to {@link sCodewords} - */ - - private static int[] computeCodewords() { - int[] cws = new int[4096]; - //iterate over all valid datawords - for (int i = 0; i < 4096; i++) { - //multiply dataword by generator matrix - int cw = 0; - for (int j = 0; j < 24; j++) { - int d = i & sGenerator[j]; - int p = Integer.bitCount(d); - cw = (cw << 1) | (p & 1); - } - //store resulting codeword - cws[i] = cw; - } - return cws; - } - - /** - * Generates error array. - * - * @return an array for assignment to {@link sErrors} - */ - - private static int[] computeErrors() { - int[] errors = new int[4096]; - //fill array with -1 (indicates that error cannot be corrected - Arrays.fill(errors, -1); - - //record syndrome for zero error (valid) word - { - int error = 0; - int syn = syndrome(error); - errors[syn] = error; - } - - //record syndrome for each single error word - for (int i = 0; i < 24; i++) { - int error = 1 << i; - int syn = syndrome(error); - errors[syn] = error; - } - - //record syndrome for each double error word - for (int i = 1; i < 24; i++) { - for (int j = 0; j < i; j++) { - int error = (1 << i) | (1 << j); - int syn = syndrome(error); - errors[syn] = error; - } - } - - //record syndrome for each triple error word - for (int i = 2; i < 24; i++) { - for (int j = 1; j < i; j++) { - for (int k = 0; k < j; k++) { - int error = (1 << i) | (1 << j) | (1 << k); - int syn = syndrome(error); - errors[syn] = error; - } - } - } - - //code can't resolve quadruple errors - return errors; - } - - /** - * Encodes a 12 bit data word into a codeword. The 12 bits must be in the - * least significant positions and all other supplied bits must be zero. - * - * @param data a 12 bit data word - * @return the 24 bit code word - */ - - public static int encode(final int data) { - return sCodewords[data]; - } - - /** - * Computes the syndrome for the supplied codeword. The 24 bits must be in - * the least significant positions. - * - * @param word a candidate code word - * @return the syndrome for the supplied word - */ - - public static int syndrome(final int word) { - //multiply codeword by the check matrix - int syndrome = 0; - for (int j = 0; j < 12; j++) { - int d = word & sCheck[j]; - int p = Integer.bitCount(d); - syndrome = (syndrome << 1) | (p & 1); - } - return syndrome; - } - - /** - * Whether the supplied candidate code word is a valid code word. The 24 - * bits must be in the least significant positions and all other supplied - * bits must be zero. - * - * @param word the candidate code word - * @return true iff the supplied word is a valid codeword - */ - - public static boolean isCodeword(final int word) { - //optimization - is it worth it? - int w = Integer.bitCount(word); - if (w != 0 && w != 8 && w != 12 && w != 16 && w != 24) return false; - return syndrome(word) == 0; - } - - /** - * Decodes a valid code word into a dataword. - * - * @param codeword a valid code word - * @return the corresponding data word - */ - public static int decodeWord(final int codeword) { - return (codeword >> 12) & MASK; - } - - /** - * Attempts to correct and decode a codeword. The 24 bits must be in the - * least significant positions and all other supplied bits must be zero. - * NOTE: for codewords with four errors, this method does not attempt any correction - * - * @param word a word to be decoded - * @return a decoded and possibly corrected data word - */ - - public static int correctAndDecode(final int word) { - int err = sErrors[ syndrome(word) ]; - //for 4 errors we currently just give up!! - return err <= 0 ? decodeWord(word) : decodeWord(word ^ err); - } - - private float[] variance; - - // constructor - - /** - * Cannot be instantiated. - */ - - public Golay() { } - - - public static void main(String[] args) { - Random r = new Random(); - int d = 24; - - Golay sp = new Golay(); - MurmurHash hash = new MurmurHash(Integer.MAX_VALUE); - float testResolution = 10000f; - - for (int i = 0; i < 300; i++) { - int ct = 0; - float distavg = 0.0f; - for (int j = 0; j < testResolution; j++) { - float p1[] = new float[d]; - float p2[] = new float[d]; - - // generate a vector - for (int k = 0; k < d; k++) { - p1[k] = r.nextFloat() * 2 - 1; - p2[k] = (float) (p1[k] + r.nextGaussian() - * ((float) i / 1000f)); - } - float dist = VectorUtil.distance(p1, p2); - distavg += dist; - - long hp1 = hash.hash(sp.decode(p1)); - long hp2 = hash.hash(sp.decode(p2)); - - ct+=(hp2==hp1)?1:0; - - } - System.out.println(distavg / testResolution + "\t" + (float) ct - / testResolution); - } - } - -// float varTot = 1.0f; - @Override - public long[] decode(float[] p1) { - int codeword = 0; - if(p1[0]>0)codeword+=1; - for(int i=1;i<24;i++){ - codeword<<=1; - if(p1[i]>0)codeword+=1; - } - return new long[]{correctAndDecode(codeword)}; - } - -// @Override -// public void setVariance(float[] parameterObject) { -// variance = parameterObject; -// for(int i = 0 ; i - float[] rndBs; - // Vector> - float[][] stableArray; - - public PsdLSH(int M, int L, int D, int T, float W) { - this.M = M; - this.L = L; - this.T = T; - this.W = W; - this.D = D; - bits = (int) Math.ceil(Math.log(M) / Math.log(2)); - rndBs = new float[L]; - stableArray = new float[L][D]; - initialize(); - } - - public PsdLSH() { - M = 256; - L = 4; - T = GAUSSIAN; - W = 2f; - D = 32; - bits = (int) Math.ceil(Math.log(M) / Math.log(2)); - rndBs = new float[L]; - stableArray = new float[L][D]; - ; - initialize(); - } - - public PsdLSH(int psdtype, int innerDecoderMultiplier) { - M = 256; - L = 4; - T = psdtype; - if (psdtype == LEVY) - W = 2f; - if (psdtype == GAUSSIAN) - W = 1f; - if (psdtype == CAUCHY) - W = 2f; - D = innerDecoderMultiplier; - - bits = (int) Math.ceil(Math.log(M) / Math.log(2)); - rndBs = new float[L]; - stableArray = new float[L][D]; - ; - initialize(); - } - - private void initialize() { - - Random rng = new Random(); - - switch (T) { - case 0: { - LevyDistribution ld = new LevyDistribution(0, 1); - for (int l = 0; l < L; l++) { - int d = 0; - while (d < D) { - stableArray[l][d] = (float) ld.sample(); - if (stableArray[l][d] < 3f && stableArray[l][d] > -3f) { - d++; - } - } - rndBs[l] = rng.nextFloat() * W; - } - return; - } - - case 1: { - CauchyDistribution cd = new CauchyDistribution(); - - for (int l = 0; l < L; l++) { - int d = 0; - while (d < D) { - stableArray[l][d] = (float) cd.sample(); - if (stableArray[l][d] < 3f && stableArray[l][d] > -3f) { - d++; - } - } - - rndBs[l] = rng.nextFloat() * W; - } - return; - } - case 2: { - for (int l = 0; l < L; l++) { - for (int d = 0; d < D; d++) { - stableArray[l][d] = (float) rng.nextGaussian(); - } - rndBs[l] = rng.nextFloat() * W; - } - return; - } - default: { - return; - } - } - } - - long[] hash(float[] v) { - - long[] hashVal = new long[1]; - // long hashVal = 0; - int tmp; - for (int l = 0; l < L; l++) { - // dot product with stable distribution - float sum = rndBs[l]; - for (int d = 0; d < D; d++) { - sum += v[d] * stableArray[l][d]; - } - tmp = ((int) ((sum) / W)); - tmp %= M; - // shift negative number to the other side - hashVal[0] += tmp; - hashVal[0] <<= this.bits; - } - return hashVal; - } - - public static void main(String[] args) { - Random r = new Random(); - // int M = 256; - // int L = 8; - // int T = LEVY; - // float W = 1f; - int d = 24; - - PsdLSH sp = new PsdLSH(); - - // MultiDecoder sp = new MultiDecoder( d, e8); - MurmurHash hash = new MurmurHash(Integer.MAX_VALUE); - float testResolution = 10000f; - - HashMap ctmap = new HashMap(); - - for (int i = 0; i < 400; i++) { - int ct = 0; - float distavg = 0.0f; - for (int j = 0; j < testResolution; j++) { - float p1[] = new float[d]; - float p2[] = new float[d]; - - // generate a vector - for (int k = 0; k < d; k++) { - p1[k] = r.nextFloat() * 2 - 1f; - p2[k] = (float) (p1[k] + r.nextGaussian() - * ((float) i / 1000f)); - } - float dist = VectorUtil.distance(p1, p2); - distavg += dist; - long[] l1 = sp.decode(p1); - long[] l2 = sp.decode(p2); - - ctmap.put(l1[0], - ctmap.containsKey(l1[0]) ? 1 + ctmap.get(l1[0]) : 1); - - long hp1 = hash.hash(l1); - long hp2 = hash.hash(l2); - - // ctmap.put(hp1,ctmap.containsKey(hp1)?1+ctmap.get(hp1):1); - - ct += (hp2 == hp1) ? 1 : 0; - - } - - System.out.println(distavg / testResolution + "\t" + (float) ct - / testResolution); - } - } - -// @Override -// public void setVariance(float[] parameterObject) { -// this.variance = parameterObject; -// } - - @Override - public int getDimensionality() { - return D; - } - - @Override - public long[] decode(float[] f) { - return hash(f); - } - - @Override - public float getErrorRadius() { - return 1; - } - - @Override - public float getDistance() { - return 0; - } - - @Override - public boolean selfScaling() { - return true; - } - - @Override - public void setCounter(Countable counter) { - // TODO Auto-generated method stub - - } - -// @Override -// public float[] getVariance() { -// return this.variance; -// } - -} diff --git a/src/main/java/edu/uc/rphash/frequentItemSet/KHHCentroidCounterPush.java b/src/main/java/edu/uc/rphash/frequentItemSet/KHHCentroidCounterPush.java deleted file mode 100644 index 819135a..0000000 --- a/src/main/java/edu/uc/rphash/frequentItemSet/KHHCentroidCounterPush.java +++ /dev/null @@ -1,56 +0,0 @@ -package edu.uc.rphash.frequentItemSet; - -import java.util.Iterator; -import java.util.List; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.knee.KneeAlgorithm; - -public class KHHCentroidCounterPush extends KHHCentroidCounter { - - int estimatedKnee = 0; - KneeAlgorithm kne; - - public KHHCentroidCounterPush(float decay, KneeAlgorithm kne) { - super(1000, decay); - this.kne = kne; - } - - /* - * (non-Javadoc) - * - * @see edu.uc.rphash.frequentItemSet.KHHCentroidCounter#getTop() - */ - @Override - public List getTop() { - return super.getTop(); - } - - /** - * @see edu.uc.rphash.frequentItemSet.KHHCentroidCounter#add(edu.uc.rphash.Centroid) - * This method adds a new vector to the khhcounter and performs knee - * finding on the khhset - * @param the - * cluster to be added c - * @return the estimated number of clusters using the provided KneeAlgorithm - * if the estimation changes or -1 if it does not - */ - public int addAndUpdate(Centroid c) { - super.add(c); - // check for new clusters - int size = frequentItems.values().size(); - float[] counts = new float[size]; - Iterator it = frequentItems.values().iterator(); - for (int i = 0; it.hasNext(); i++) { - counts[i] = it.next().getCount(); - } - int tmpknee = kne.findKnee(counts); - if (tmpknee != estimatedKnee) { - estimatedKnee = tmpknee; - return estimatedKnee; - } else { - return -1; - } - } - -} diff --git a/src/main/java/edu/uc/rphash/knee/BiggestMergeKnee.java b/src/main/java/edu/uc/rphash/knee/BiggestMergeKnee.java deleted file mode 100644 index 2980d0d..0000000 --- a/src/main/java/edu/uc/rphash/knee/BiggestMergeKnee.java +++ /dev/null @@ -1,38 +0,0 @@ -package edu.uc.rphash.knee; - -import edu.uc.rphash.util.VectorUtil; - -public class BiggestMergeKnee implements KneeAlgorithm { - - @Override - public int findKnee(float[] data) { - - return data.length/2; - } - - - - - /** - * this function creates a linear model y=alpha*x+beta for the given data - * series x,y. - */ - float[] linest(float[] y) { - - int n = y.length; - float[] x = new float[n]; - for(int i = 0;imaxdist){ - maxdist = tmpdist; - argmax = i; - } - } - return argmax; - } - - - - /** - * this function creates a linear model y=alpha*x+beta for the given data - * series x,y. - */ - float[] linest(float[] y) { - - int n = y.length; - float[] x = new float[n]; - for(int i = 0;i>> 56); - s2[ct++] = (byte) (d >>> 48); - s2[ct++] = (byte) (d >>> 40); - s2[ct++] = (byte) (d >>> 32); - s2[ct++] = (byte) (d >>> 24); - s2[ct++] = (byte) (d >>> 16); - s2[ct++] = (byte) (d >>> 8); - s2[ct++] = (byte) (d); - - return computeCWowIntHash(s2, 0) % tablesize; - } - - @Override - public long hash(long[] s) { - byte[] s2 = new byte[s.length * 8]; - int ct = 0; - for (long d : s) { - s2[ct++] = (byte) (d >>> 56); - s2[ct++] = (byte) (d >>> 48); - s2[ct++] = (byte) (d >>> 40); - s2[ct++] = (byte) (d >>> 32); - s2[ct++] = (byte) (d >>> 24); - s2[ct++] = (byte) (d >>> 16); - s2[ct++] = (byte) (d >>> 8); - s2[ct++] = (byte) (d); - } - return computeCWowIntHash(s2, 0) % tablesize; - } - - public final static int CWOW_32_M = 0x57559429; - public final static int CWOW_32_N = 0x5052acdb; - public static final long LONG_LO_MASK = 0x00000000FFFFFFFFL; - - /** gather an int from the specified index into the byte array */ - public static final int gatherIntLE(byte[] data, int index) { - int i = data[index] & 0xFF; - i |= (data[++index] & 0xFF) << 8; - i |= (data[++index] & 0xFF) << 16; - i |= (data[++index] << 24); - return i; - } - - public static final int gatherPartialIntLE(byte[] data, int index, - int available) { - int i = data[index] & 0xFF; - if (available > 1) { - i |= (data[++index] & 0xFF) << 8; - if (available > 2) { - i |= (data[++index] & 0xFF) << 16; - } - } - return i; - } - - public int computeCWowIntHash(byte[] data, int seed) { - final int length = data.length; - /* cwfold( a, b, lo, hi ): */ - /* p = (u32)(a) * (u64)(b); lo ^=(u32)p; hi ^= (u32)(p >> 32) */ - /* cwmixa( in ): cwfold( in, m, k, h ) */ - /* cwmixb( in ): cwfold( in, n, h, k ) */ - int hVal = seed; - int k = length + seed + CWOW_32_N; - long p = 0; - int pos = 0; - int len = length; - while (len >= 8) { - int i1 = gatherIntLE(data, pos); - int i2 = gatherIntLE(data, pos + 4); - /* cwmixb(i1) = cwfold( i1, N, hVal, k ) */ - p = i1 * (long) CWOW_32_N; - k ^= p & LONG_LO_MASK; - hVal ^= (p >> 32); - /* cwmixa(i2) = cwfold( i2, M, k, hVal ) */ - p = i2 * (long) CWOW_32_M; - hVal ^= p & LONG_LO_MASK; - k ^= (p >> 32); - pos += 8; - len -= 8; - } - if (len >= 4) { - int i1 = gatherIntLE(data, pos); - /* cwmixb(i1) = cwfold( i1, N, hVal, k ) */ - p = i1 * (long) CWOW_32_N; - k ^= p & LONG_LO_MASK; - hVal ^= (p >> 32); - pos += 4; - len -= 4; - } - if (len > 0) { - int i1 = gatherPartialIntLE(data, pos, len); - /* cwmixb(i1) = cwfold( i1, N, hVal, k ) */ - p = (i1 & ((1 << (len * 8)) - 1)) * (long) CWOW_32_M; - hVal ^= p & LONG_LO_MASK; - k ^= (p >> 32); - } - p = (hVal ^ (k + CWOW_32_N)) * (long) CWOW_32_N; - k ^= p & LONG_LO_MASK; - hVal ^= (p >> 32); - hVal ^= k; - return hVal; - } - -} diff --git a/src/main/java/edu/uc/rphash/tests/ScalabilityTest.java b/src/main/java/edu/uc/rphash/tests/ScalabilityTest.java deleted file mode 100644 index b73826d..0000000 --- a/src/main/java/edu/uc/rphash/tests/ScalabilityTest.java +++ /dev/null @@ -1,130 +0,0 @@ -package edu.uc.rphash.tests; - -import java.util.ArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.Future; - -import edu.uc.rphash.RPHashAdaptive2PassParallel; -import edu.uc.rphash.RPHashSimpleParallel; -import edu.uc.rphash.RPHashStream; -import edu.uc.rphash.concurrent.VectorLevelConcurrency; -import edu.uc.rphash.tests.generators.GenerateStreamData; - -public class ScalabilityTest { - - public static long rphashstream(ArrayList vecsAndNoiseInThisRound, - int i, int k, GenerateStreamData gen1) { - RPHashStream rphit = new RPHashStream(k, gen1, i); - long timestart = System.nanoTime(); - //vecsAndNoiseInThisRound.parallelStream().map(vec-> - // VectorLevelConcurrency.computeSequential(vec, rphit.lshfuncs.get(0), rphit.is.get(0), rphit.getParam())); - for (float[] v : vecsAndNoiseInThisRound) - rphit.addVectorOnlineStep(v); - rphit.getCentroidsOfflineStep(); - - return System.nanoTime() - timestart; - } - - public static long rphashsimple(ArrayList vecsAndNoiseInThisRound, - int i, int k) { - RPHashSimpleParallel rphit = new RPHashSimpleParallel( - vecsAndNoiseInThisRound, k, i); - - long timestart = System.nanoTime(); - rphit.mapreduce1(); - rphit.mapreduce2(); - return System.nanoTime() - timestart; - } - - public static long rphashadaptive( - ArrayList vecsAndNoiseInThisRound, int i, int k) { - - RPHashAdaptive2PassParallel rphit = new RPHashAdaptive2PassParallel( - vecsAndNoiseInThisRound, k, i); - - long timestart = System.nanoTime(); - rphit.run(); - return System.nanoTime() - timestart; - } - - public static void scalability(int n) { - int k = 10; - int d = 1000; - float var = 1f; - Runtime rt = Runtime.getRuntime(); - // Random r = new Random(); - int NUM_Procs = rt.availableProcessors(); - - GenerateStreamData gen1 = new GenerateStreamData(k, d, var, 11331313); - - ArrayList vecsAndNoiseInThisRound = new ArrayList(n); - - // generate data in parallel - vecsAndNoiseInThisRound = gen1.genParallel(n); - - System.out.println(vecsAndNoiseInThisRound.size()); - System.out.printf("Threads\tSimple\tStream\tAdaptive\n"); - - long timesimple = 0, timeadaptive = 0, timestream = 0; - - for (int i = 1; i <= NUM_Procs; i++) { - - try { - //mix up the order - if(i%3==0){ - System.gc(); - Thread.sleep(1000); - timesimple = rphashsimple(vecsAndNoiseInThisRound, i, k); - System.gc(); - Thread.sleep(1000); - timeadaptive = rphashadaptive(vecsAndNoiseInThisRound, i, k); - System.gc(); - Thread.sleep(1000); - timestream = rphashstream(vecsAndNoiseInThisRound, i, k, gen1); - } - - if(i%3==1){ - System.gc(); - Thread.sleep(1000); - timeadaptive = rphashadaptive(vecsAndNoiseInThisRound, i, k); - System.gc(); - Thread.sleep(1000); - timesimple = rphashsimple(vecsAndNoiseInThisRound, i, k); - System.gc(); - Thread.sleep(1000); - timestream = rphashstream(vecsAndNoiseInThisRound, i, k, gen1); - } - - if(i%3==2){ - System.gc(); - Thread.sleep(1000); - timesimple = rphashsimple(vecsAndNoiseInThisRound, i, k); - System.gc(); - Thread.sleep(1000); - timestream = rphashstream(vecsAndNoiseInThisRound, i, k, gen1); - System.gc(); - Thread.sleep(1000); - timeadaptive = rphashadaptive(vecsAndNoiseInThisRound, i, k); - } - - System.out.printf("%d\t%.6f\t%.6f\t%.6f\n", i, - timesimple / 1e9f, timestream / 1e9f, - timeadaptive / 1e9f); - - } catch (Exception e) { - e.printStackTrace(); - System.out.println("Exception at Proc:" + String.valueOf(i)); - System.out.printf("%d\t%.6f\t%.6f\t%.6f\n", i, - timesimple / 1e9f, timestream / 1e9f, - timeadaptive / 1e9f); - - } - } - } - - public static void main(String[] args) throws InterruptedException { - ScalabilityTest.scalability(Integer.parseInt(args[0])); - - } -} diff --git a/src/main/java/edu/uc/rphash/tests/TestRPhash.java b/src/main/java/edu/uc/rphash/tests/TestRPhash.java index 2ff2498..6a6d8b3 100644 --- a/src/main/java/edu/uc/rphash/tests/TestRPhash.java +++ b/src/main/java/edu/uc/rphash/tests/TestRPhash.java @@ -5,7 +5,7 @@ import edu.uc.rphash.Centroid; import edu.uc.rphash.RPHashSimple; -import edu.uc.rphash.tests.clusterers.LloydIterativeKmeans; + import edu.uc.rphash.tests.generators.GenerateData; import edu.uc.rphash.util.VectorUtil; @@ -25,7 +25,7 @@ static void testRPHash(int k, int n,int d,float variance,int projdim){ long startTime = System.nanoTime(); - List M = ( new LloydIterativeKmeans(k,gen.data(),projdim)).getCentroids(); + long duration = (System.nanoTime() - startTime); // List aligned = VectorUtil.alignCentroids(M,gen.medoids()); diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/AdaptiveMeanShift.java b/src/main/java/edu/uc/rphash/tests/clusterers/AdaptiveMeanShift.java deleted file mode 100644 index 122dac3..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/AdaptiveMeanShift.java +++ /dev/null @@ -1,432 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.Clusterer; -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.Readers.SimpleArrayReader; -import edu.uc.rphash.kdtree.KDTreeNN; -import edu.uc.rphash.kdtree.naiveNN; -import edu.uc.rphash.lsh.LSHkNN; -import edu.uc.rphash.tests.generators.GenerateData; -import edu.uc.rphash.util.VectorUtil; - -/* Adaptive Mean Shift (AMS) Algorithm - * - * - * Mean Shift algorithm based on methods described by Fukunaga and Hostetler - * 'Estimation of the Gradient of a Density Function, with Applications - * in Pattern Recognition' ( - * - * http://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=1055330 - * - * - * - * Additional Kernel and Optimizations described by Cheng - * 'Mean Shift, Mode Seeking, and Clustering' (1995) - * - * http://dl.acm.org/citation.cfm?id=628711 - * - * - * - * Adaptive Mean Shift algorithm based on ... - * - * - */ - - -//TODO: Add labels to points for centroids -//TODO: add weights to centroid merging -> rphash (cardinality) -//TODO: windowMode -> Sample Point Estimator - -final class cStore{ - public int count; - public float[] centroid; - public float wcsse = 0; - public Centroid cent; - - public void addPoint(float[] point){ - this.count++; - this.wcsse += VectorUtil.distance(point, centroid); - this.cent.setCount(this.count); - //TODO: this.cent.setWCSS(this.wcsse); - } - - public cStore(float[] centroid){ - this.count = 0; - this.cent = new Centroid(centroid,0); - this.centroid = centroid; - this.wcsse = 0; - } - - public cStore(float[] window, float[] point) { - // TODO Auto-generated constructor stub - this.count = 0; - this.cent = new Centroid(window,0); - this.centroid = window; - this.wcsse = VectorUtil.distance(point, window); - } - -} - - -public class AdaptiveMeanShift implements Clusterer { - - List data; //global data storage - List centroids; //global centroid storage - private RPHashObject so; - private List cs; - - //Parameters - double h = 1; // bandwidth - - int kernelMode = 0; // mode (0:uniform; 1:gaussian) - - int windowMode = 1; // Determine how to perform the Adaptive Window - // 0 - No adaptivity; Basic Mean Shift - // 1 - Balloon Estimator - // 2 - Sample Point Estimator (TODO) - - int knnAlg = 2; //Determine what KNN algorithm to use - // 0 - kNN Naive - // 1 - kNN LSH - // 2 - KD-TREE kNN - - int k = 5; //Number of KNN points for adaptive window - - Clusterer weightClusters = null; - - - static int maxiters = 10000; //Max iterations before breaking search for convergence - float convergeValue = (float) 0.00001; //maximum change in each dimension to 'converge' - float blurPercent = (float) 2; //Amount to blur centroids to group similar Floats - - //TEST Parameters: - boolean debug = false; //Control Debug Output - boolean minimalOutput = true; //Print the minimal final output (pretty print) - boolean printCentroids = true; //Print out centroids (not pretty) - Set cent = new HashSet(); //Storage for grouping the clusters - - public void setMode(int mode){ this.kernelMode = mode; } - - public void setH(double h) { this.h = h; } - - public void setWinMode(int winMode){ this.windowMode = winMode; } - - public List getData() { return data; } - - public void setRawData(List data){ this.data = data; } - - - public AdaptiveMeanShift(){ - this.centroids = new ArrayList(); - this.cs = new ArrayList(); - } - - public AdaptiveMeanShift(int k, List data){ - this.k = k; - this.data = data; - this.centroids = new ArrayList(); - } - - public AdaptiveMeanShift(int h, int windowMode, int kernelMode, int k, List data){ - this.h = h; - this.windowMode = windowMode; - this.kernelMode = kernelMode; - this.k = k; - this.data = data; - this.centroids = new ArrayList(); - this.cs = new ArrayList(); - } - - public AdaptiveMeanShift(int h, int windowMode, int kernelMode, int k, List data, Clusterer c){ - this.h = h; - this.weightClusters = c; - this.windowMode = windowMode; - this.kernelMode = kernelMode; - this.k = k; - this.data = data; - this.centroids = new ArrayList(); - this.cs = new ArrayList(); - } - - public float calcMode(float curWindow, float workingData){ - float mPoint = 0; - float kern = 0; - - if (kernelMode == 0) //Uniform - mPoint = workingData; - else if (kernelMode == 1){ //Gaussian - float c = (float) (1.0/Math.pow(h,2)); - kern = (float) Math.exp(-c * Math.pow(workingData - curWindow, 2)); - mPoint = (float) kern * (workingData - curWindow); - } - - return mPoint; - } - - public void adaptH(List data, int curPoint, LSHkNN knnHandle, KDTreeNN kdHandle, naiveNN naiveHandle){ - if(windowMode == 0) //No adaptivity - return; - else if(windowMode == 1){ //Balloon - if(knnAlg == 0){ - h = Math.sqrt(naiveHandle.getNNEuc(k, data.get(curPoint))); - printDebug("naiveH: " + h); - } - if(knnAlg == 1){ - List retData = knnHandle.knn(k, data.get(curPoint)); - h = VectorUtil.distance(retData.get(retData.size() - 1),data.get(curPoint)); - printDebug("LSHH: " + h); - } - if(knnAlg == 2){ - h = Math.sqrt(kdHandle.treeNNEuc(k, data.get(curPoint))); - printDebug("KDH: " + h + "\n"); - } - - return; - } - else if(windowMode == 2){ //KNN sample point estimator - return; - } - } - - - - public void cluster(List data){ - LSHkNN knnHandle = null; - KDTreeNN kdHandle = null; - naiveNN naiveHandle = null; - if(windowMode == 1){ - if(knnAlg == 0){ - naiveHandle = new naiveNN(data); - } - if(knnAlg == 1){ - knnHandle = new LSHkNN(data.get(0).length,5); - knnHandle.createDB(data); - } - if(knnAlg == 2){ - kdHandle = new KDTreeNN(); - kdHandle.createTree(data); - } - } - - - for(int i = 0; i < data.size(); i++){ - - float[] curWindow = new float[data.get(0).length]; - float[] bufWindow = new float[data.get(0).length]; - boolean converge = false; - int m = 0; - int winCount = 0; - - for(int t = 0; t < data.get(0).length; t++){ - curWindow = data.get(i).clone(); - } - - adaptH(data, i, knnHandle, kdHandle, naiveHandle); - - while((!converge) && (m < maxiters)){ - m++; - bufWindow = curWindow.clone(); - - for(int t = 0; t < data.get(0).length; t++){ - curWindow[t] = (float) 0; - } - - for(int x = 0; x < data.size(); x++){ - - if(VectorUtil.distance(bufWindow, data.get(x)) <= h){ - winCount++; - - for(int n = 0; n < data.get(x).length; n++){ - curWindow[n] = curWindow[n] + calcMode(bufWindow[n], data.get(x)[n]); - } - } - } - - if(winCount > 0){ - boolean convergeTest = true; - - for(int y = 0; y < curWindow.length; y++){ - if(curWindow[y] >= convergeValue) - convergeTest = false; - } - - if(kernelMode == 0){ - for(int y = 0; y < curWindow.length; y++){ - curWindow[y] = curWindow[y] / winCount; - } - } - if(kernelMode >= 1){ - for(int y = 0; y < curWindow.length; y++){ - curWindow[y] = curWindow[y] / winCount; - curWindow[y] = bufWindow[y] + curWindow[y]; - printDebug("New Window: " + curWindow[y]); - } - printDebug("_______________________________________"); - } - - - //Check for convergence - if(Arrays.equals(curWindow,bufWindow) || convergeTest){ - boolean add = true; - if(centroids.indexOf(curWindow) >= 0){ - add = false; - } - add = checkAllCentroids(curWindow, data.get(i)); - - if(add){ - String str = ""; - for(int j = 0; j < curWindow.length; j++){str += Float.toString(curWindow[j]) + ",";} - cent.add(str + "\n"); - } - - converge = true; - } - bufWindow = curWindow.clone(); - } - - m = 0; - winCount = 0; - } - } - - for(cStore cen: cs){ - Centroid it = new Centroid(cen.centroid, 0); - it.setCount(cen.count); - //TODO: it.setWCSS(cen.wcsse); - centroids.add(it); - - } - } - - - public boolean checkAllCentroids(float[] window, float[] point){ - float[] centroid; - for(cStore cz : cs){ - centroid = cz.centroid; - double percentDiff = 0; - - for(int z = 0; z < centroid.length; z++){ - percentDiff = percentDiff + Math.abs(1-(centroid[z] / window[z])); - } - - percentDiff = percentDiff / centroid.length; - - if(percentDiff < blurPercent){ - cz.addPoint(point); - return false; - } - - } - - cs.add(new cStore(window, point)); - return true; - } - - - void run(){ - if(this.weightClusters != null){ - //this.weightClusters.setData(this.data); - } - - cluster(this.data); - } - - - public void printDebug(String s){ - if(debug) - System.out.println(s); - } - - - public static void main(String[] args){ - int genClusters = 3; - int genRowsPerCluster =100; - int genColumns = 100; - - AdaptiveMeanShift ams = new AdaptiveMeanShift(); - - if(ams.data == null){ - GenerateData gen = new GenerateData(genClusters,genRowsPerCluster, genColumns); - ams.data = gen.data; - } - - ams.run(); - if(ams.printCentroids){ - System.out.println("Centroid Count: " + ams.centroids.size()); - for(Centroid c: ams.centroids){ - System.out.println("WCSS = " + c.getWCSS()); - System.out.print("Cent = "); - for(int z = 0; z < c.centroid().length; z++) - System.out.print(c.centroid()[z] + ","); - System.out.println("\n\n"); - } - } - if(ams.minimalOutput){ - System.out.println("\n\nh: " + ams.h); - System.out.println("Kernel Mode: " + ams.kernelMode); - System.out.println("Window Mode: " + ams.windowMode); - System.out.println("k (KNN): " + ams.k + "\n"); - System.out.println("Number of Clusters: " + ams.cent.size() + "\n"); - System.out.println(ams.cent.toString().replaceAll(", ", " ")); - } - - System.out.println("\n\nDone!"); - } - - - @Override - public List getCentroids() { - if(this.centroids.size() == 0) - run(); - return this.centroids; - } - - @Override - public RPHashObject getParam() { - so = new SimpleArrayReader(this.data, k); - return so; - } - - @Override - public void setK(int getk) { - this.k = getk; - } - - @Override - public void setWeights(List counts) { - // TODO Auto-generated method stub - if(data != null) { - - } - - - } - - @Override - public void setData(List centroids) { - ArrayList data = new ArrayList(centroids.size()); - for(Centroid c : centroids)data.add(c.centroid()); - setRawData(data); - } - - @Override - public void reset(int randomseed) { - // TODO Auto-generated method stub - this.centroids = null; - - } - - @Override - public boolean setMultiRun(int runs) { - // Return true to ignore multi-run (deterministic) - return true; - } - -} \ No newline at end of file diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative.java b/src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative.java deleted file mode 100644 index bedf99d..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative.java +++ /dev/null @@ -1,151 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.List; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.Clusterer; -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.tests.generators.GenerateData; -import edu.uc.rphash.util.VectorUtil; - -public class Agglomerative implements Clusterer{ - - int k; - List clusters; - List data; - float[][] distances; - List counts; - public Agglomerative() - { - - } - public Agglomerative(int k, List data) - { - this.k = k; - this.data = data; - this.clusters = null; - counts = new ArrayList(); - for(int i = 0;i data){ - float[][] distances = new float[data.size()][data.size()]; - - for(int i = 0 ; i < data.size();i++) - { - for(int j = 0; j < data.size();j++) - distances[i][j] =VectorUtil.distance(data.get(i), data.get(j)); - } - return distances; - } - - - private float[] avgVector(float[] u, float[] v, Float float1, Float float2){ - float[] w = new float[u.length]; - for(int i = 0 ;i < u.length;i++)w[i] = (u[i]*float1+v[i]*float2)/(float1+float2); - return w; - } - - private void merge() - { - float minimum = 1000000f; - int mini = 0; - int minj = 0; - int i = 0 ; - for(float[] l : distances) - { - for(int j = 0; j < data.size();j++){ - if(l[j]k) - merge(); - } - - public static void main(String[] args){ - GenerateData gen = new GenerateData(3,500,2); - List data =gen.data; - float[][] dists = distanceArray(data); -// double[] weights = new double[data.size()]; - - - String[] s = new String[dists.length]; - for(int i = 0;i< dists.length;i++)s[i] = String.valueOf(i); - - Agglomerative agl = new Agglomerative(3, data); - agl.run(); - for(float[] cent: gen.getMedoids()){ - for(float f : cent)System.out.print(f+" "); - System.out.println(); - } - System.out.println("computed"); - - for(Centroid cent: agl.getCentroids()){ - for(float f : cent.centroid())System.out.print(f+" "); - System.out.println(); - } - - } - - @Override - public List getCentroids() { - if(clusters==null)run(); - List cents = new ArrayList<>(clusters.size()); - for(float[] v : this.clusters)cents.add(new Centroid(v,0)); - return cents; - } - - @Override - public void reset(int randomseed) { - clusters = null; - } - - - @Override - public RPHashObject getParam() { - // TODO Auto-generated method stub - return null; - } - - @Override - public void setWeights(List counts) { - //this.counts = counts; - counts = new ArrayList(); - } - @Override - public void setK(int getk) { - this.k = getk; - } - @Override - public void setData(List centroids) { - this.data = new ArrayList(centroids.size()); - for(Centroid c : centroids) data.add(c.centroid()); - } - @Override - public void setRawData(List centroids) { - this.data = centroids; - } - - @Override - public boolean setMultiRun(int runs) { - //agglomerative is deterministic running multiple times is moot - return true; - } -} diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative2.java b/src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative2.java deleted file mode 100644 index ad5cd4a..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/Agglomerative2.java +++ /dev/null @@ -1,372 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.TreeSet; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.Clusterer; -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.tests.StatTests; -import edu.uc.rphash.tests.generators.GenerateData; -import edu.uc.rphash.util.VectorUtil; - -public class Agglomerative2 implements Clusterer { - private class DistAndVector implements Comparable { - Float dist; - Integer vec; - - @Override - public int compareTo(DistAndVector o) { - // if (equals(o)) - // return 0; - if (dist.floatValue() == o.dist.floatValue()) - return 1; - if (dist.floatValue() < o.dist.floatValue()) - return -1; - return 1; - - } - } - - private class PQAndVector implements Comparable { - TreeSet pq; - Integer vec; - - PQAndVector(Integer vec) { - this.pq = new TreeSet(); - this.vec = vec; - } - - // @Override - // public boolean equals(Object o) { - // - // return ((PQAndVector) o).vec.intValue() == vec.intValue(); - // } - - @Override - public int compareTo(PQAndVector r) { - - // if (equals(r)){ - // System.out.println("whatthehell"+ r.vec.intValue() + ":"+ - // this.vec.intValue()); - // return 0; - // - // } - if (pq.isEmpty()) { - return 1; - } - if (r.pq.isEmpty()) { - return -1; - } - - if (pq.first().dist == r.pq.first().dist) { - return 1; - } - return pq.first().compareTo(r.pq.first()); - } - } - - int k; - TreeSet outerpq = new TreeSet(); - List data; - float counts[]; - - private void distanceArray(List data) { - int n = data.size(); - for (int i = 0; i < n - 1; i++) { - PQAndVector innerpq = new PQAndVector(new Integer(i)); - - for (int j = i + 1; j < n; j++) { - DistAndVector dv = new DistAndVector(); - dv.dist = new Float(VectorUtil.distance(data.get(i), - data.get(j))); - dv.vec = new Integer(j); - innerpq.pq.add(dv); - } - // - // System.out.print(i+" : "); - // for(Object p: - // innerpq.pq.toArray())System.out.print(((DistAndVector)p).vec+", ");System.out.println(); - outerpq.add(innerpq); - } - // for(PQAndVector p: - // outerpq)System.out.print(p.vec+", ");System.out.println(); - - } - - private void distanceArray2(List data2,List projIDs) { - int n = data.size(); - for (int i = 0; i < n - 1; i++) { - PQAndVector innerpq = new PQAndVector(new Integer(i)); - - for (int j = i + 1; j < n; j++) { - DistAndVector dv = new DistAndVector(); - if(projIDs.get(i).equals(projIDs.get(j))){ - dv.dist = Float.MAX_VALUE; - } - else{ - dv.dist = new Float(VectorUtil.distance(data.get(i), - data.get(j))); - } - - - dv.vec = new Integer(j); - innerpq.pq.add(dv); - } - outerpq.add(innerpq); - } - } - - private void mergeAndUpdateCentroids(int newdata, int olddata) - { - float[] u = data.get(newdata); - float[] v = data.get(olddata); - float ct1 = counts[newdata]; - float ct2 = counts[olddata]; - float[] w = new float[u.length]; - for (int i = 0; i < u.length; i++) - w[i] = (u[i] * ct1 + v[i] * ct2) / (ct1 + ct2); - counts[newdata] += counts[olddata]; - data.set(newdata, w); - - } - - /** - * remove the next two nearest vectors and perform a counts weighted average - * of the vectors. put this vector in the lower of the two vector indeces. - */ - private void merge() { - // pop the queue with the nearest top vector in it - - PQAndVector innerpq = outerpq.pollFirst(); - //lower id lists are not checked for removals, check here. - while (innerpq.pq.isEmpty()) { - innerpq = outerpq.pollFirst(); - } - // pop the nearest vector - DistAndVector dv = innerpq.pq.pollFirst(); - - int newvecloc = innerpq.vec; - int olddata = dv.vec; - - Iterator it = outerpq.iterator(); - while (it.hasNext()) { - PQAndVector v = it.next(); - if (v.vec.intValue() == olddata) { - - it.remove(); - break; - } - } - - // merge the two vectors - mergeAndUpdateCentroids(newvecloc, olddata); - - PQAndVector newpq = new PQAndVector(newvecloc); - - Iterator pqit = outerpq.iterator(); - while (pqit.hasNext()) { - PQAndVector itpq = pqit.next(); - - // remove the merged vectors from all inner lists - Iterator itdv = itpq.pq.iterator(); - while (itdv.hasNext()) { - DistAndVector v = itdv.next(); - if (v.vec.intValue() == newvecloc) { - itdv.remove(); - break; - } - } - - itdv = itpq.pq.iterator(); - while (itdv.hasNext()) { - DistAndVector v = itdv.next(); - if ( v.vec.intValue() == olddata) { - itdv.remove(); - break; - } - } - } - - // System.out.println("lists after removal"); - // printlists(); - - pqit = outerpq.iterator(); - while (pqit.hasNext()) { - PQAndVector itpq = pqit.next(); - // add the new vector distance to all upper parent lists - // compute new distance to vecs who have this vector in their list - if (itpq.vec < newvecloc) { - DistAndVector dv3 = new DistAndVector(); - dv3.dist = new Float(VectorUtil.distance(data.get(newvecloc), - data.get(itpq.vec))); - - dv3.vec = new Integer(newvecloc); - - // add the updated vector to the new lists - itpq.pq.add(dv3); - } - } - - // System.out.println("lists after adding back into lower idx lists"); - // printlists(); - - pqit = outerpq.iterator(); - while (pqit.hasNext()) { - PQAndVector itpq = pqit.next(); - // add to the new merge list - if (itpq.vec > newvecloc) { - DistAndVector dv2 = new DistAndVector(); - dv2.dist = new Float(VectorUtil.distance(data.get(newvecloc), - data.get(itpq.vec))); - dv2.vec = new Integer(itpq.vec); - newpq.pq.add(dv2); - } - } - - outerpq.add(newpq); - - // System.out.println("adding merged list back"); - // printlists(); - - } - - private void printlists() { - System.out.println(); - for (Object o : outerpq.toArray()) { - System.out.print("\t" + ((PQAndVector) o).vec + " : "); - for (Object p : ((PQAndVector) o).pq.toArray()) { - System.out.print(((DistAndVector) p).vec + ", "); - } - System.out.println(); - } - } - - private void run() { - while (outerpq.size() > k) { - - merge(); - } - Iterator pqit = outerpq.iterator(); - centroids = new ArrayList(); - while (pqit.hasNext()) { - PQAndVector innerpq = pqit.next(); - centroids.add(data.get(innerpq.vec)); - } - } - - public static void main(String[] args) { - - for (int i = 0; i < 1000; i += 10) { - long avgtime = 0; - float avgdistagg = 0; - float avgdistreal = 0; - float avgdistkm = 0; - if(i!=0){ - for (int j = 0; j < 5; j++) { - GenerateData gen = new GenerateData(10, i, 10, .5f); - List data = gen.data; - - long timestart = System.currentTimeMillis(); - Clusterer km1 = new LloydIterativeKmeans(10, data); - Clusterer ag1 = new Agglomerative2(10, data); - avgdistagg+=StatTests.WCSSECentroidsFloat(ag1.getCentroids(), data); - avgdistkm+=StatTests.WCSSECentroidsFloat(km1.getCentroids(), data); - avgdistreal+=StatTests.WCSSE(gen.getMedoids(), data); - avgtime += (System.currentTimeMillis() - timestart); - } - } - System.out.println(i + "\t" + avgtime / 5+"\t"+avgdistagg/5f+"\t"+avgdistkm/5f+"\t"+avgdistreal/5f); - } - - } - - List centroids; - - @Override - public List getCentroids() { - if(centroids==null)run(); - List cents = new ArrayList<>(centroids.size()); - for(float[] v : this.centroids)cents.add(new Centroid(v,0)); - return cents; - } - - @Override - public void reset(int randomseed) { - centroids = null; - } - - @Override - public RPHashObject getParam() { - return null; - } - - public void printDistanceArray() { - for (int i = 0; i < data.size(); i++) { - for (int j = 0; j < data.size(); j++) - System.out.printf("%.2f,", - VectorUtil.distance(data.get(i), data.get(j))); - System.out.println(); - } - System.out.println(); - } - - public Agglomerative2(int k, List data) { - this.k = k; - this.data = data; - this.counts = new float[data.size()]; - for (int i = 0; i < counts.length; i++) - counts[i] = 1; - - distanceArray(data); - - } - - public Agglomerative2(int k, List data, List counts) { - this.k = k; - this.data = data; - this.counts = new float[counts.size()]; - for (int i = 0; i < counts.size(); i++) - this.counts[i] = counts.get(i); - distanceArray(data); - } - - public Agglomerative2(int k, List data, List counts,List projectionIDs) { - this.k = k; - this.data = data; - this.counts = new float[counts.size()]; - for (int i = 0; i < counts.size(); i++) - this.counts[i] = counts.get(i); - distanceArray2(data,projectionIDs); - } - - @Override - public void setWeights(List counts) { - - } - - @Override - public void setData(List centroids) { - this.data = new ArrayList(centroids.size()); - for(Centroid c : centroids) data.add(c.centroid()); - } - @Override - public void setRawData(List centroids) { - this.data = centroids; - } - - @Override - public void setK(int getk) { - this.k = getk; - } - - @Override - public boolean setMultiRun(int runs) { - //agglomerative is deterministic running multiple times is moot - return true; - } - -} diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/Kmeans.java b/src/main/java/edu/uc/rphash/tests/clusterers/Kmeans.java deleted file mode 100644 index 2f92ac1..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/Kmeans.java +++ /dev/null @@ -1,330 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.Clusterer; -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.Readers.SimpleArrayReader; -import edu.uc.rphash.projections.DBFriendlyProjection; -import edu.uc.rphash.projections.Projector; -import edu.uc.rphash.tests.generators.GenerateData; -import edu.uc.rphash.util.VectorUtil; - -import org.apache.commons.lang3.ArrayUtils; -//import org.rosuda.JRI.REXP; -//import org.rosuda.JRI.Rengine; - -public class Kmeans implements Clusterer { - int k; - int n; - List data; - public int getK() { - return k; - } - - public void setK(int k) { - this.k = k; - } - - public List getData() { - return data; - } - - @Override - public void setData(List centroids) { - this.data = new ArrayList(centroids.size()); - for(Centroid c : centroids) data.add(c.centroid()); - } - @Override - public void setRawData(List centroids) { - this.data = centroids; - } - - public List getWeights() { - return weights; - } - - public void setWeights(List weights) { - this.weights = weights; - } - - int projdim; - -// List means; -// List kmeansCentroids = new ArrayList(); - List> clusters; - List weights; -// Rengine re; -// -// public void setRengine(Rengine re) { -// this.re = re; -// } - - public Kmeans(int k, List data) { - this.k = k; - this.data = data; - this.projdim = 0; - this.clusters = null; - this.weights = new ArrayList(data.size()); - for (int i = 0; i < data.size(); i++) - weights.add(1f); -// means = null; - } - - public Kmeans(int k, List data, List weights) { - this.k = k; - this.data = data; - this.projdim = 0; - this.clusters = null; - this.weights = weights; -// means = null; - } - - public Kmeans(int k, List data, int projdim) { - this.k = k; - this.data = data; - this.projdim = projdim; - this.clusters = null; - this.weights = new ArrayList(data.size()); - for (int i = 0; i < data.size(); i++) - weights.add(1f); -// means = null; - } - -// public Kmeans(Rengine re) { -// // TODO Auto-generated constructor stub -// this.re = re; -// } - - /* - public float[] computerCentroid(List vectors, List data) { - int d = data.get(0).length; - float[] centroid = new float[d]; - - for (int i = 0; i < d; i++) - centroid[i] = 0.0f; - - float w_total = 0f; - for (Integer v : vectors) { - w_total += weights.get(v); - } - - for (Integer v : vectors) { - float[] vec = data.get(v); - float weight = (float) weights.get(v) / (float) w_total; - for (int i = 0; i < d; i++) - centroid[i] += (vec[i] * weight); - } - return centroid; - } - - - ArrayList weightTotals; - - void updateMeans(List data) { - weightTotals = new ArrayList(); - if (means == null) { - means = new ArrayList(); - for (int i = 0; i < k; i++) - means.add(computerCentroid(clusters.get(i), data)); - } - for (int i = 0; i < k; i++) - means.set(i, computerCentroid(clusters.get(i), data)); - } - - int assignClusters(List data) { - int swaps = 0; - List> newClusters = new ArrayList>(); - for (int j = 0; j < k; j++) - newClusters.add(new ArrayList()); - - for (int clusterid = 0; clusterid < k; clusterid++) { - - for (Integer member : clusters.get(clusterid)) { - - int nearest = VectorUtil.findNearestDistance(data.get(member), - means); - newClusters.get(nearest).add(member); - if (nearest != clusterid) - swaps++; - } - - } - clusters = newClusters; - return swaps; - } - - - private void run() { - int maxiters = 1000; - int swaps = 2; - this.n = this.data.size(); - ArrayList workingdata = new ArrayList(); - // stuff for projected kmeans - Projector p = null; - Random r = new Random(); - if (projdim != 0) - p = new DBFriendlyProjection(this.data.get(0).length, projdim, - r.nextInt()); - for (float[] v : this.data) { - if (p != null) { - workingdata.add(p.project(v)); - } else - workingdata.add(v); - } - - int maxout = 0; - //loop until there are no more nullsets - boolean nullset = false; - do { - this.clusters = new ArrayList>(k); - // seed data with new clusters - ArrayList shufflelist = new ArrayList(data.size()); - for (int i = 0; i < data.size(); i++) - shufflelist.add(i); - - for (int i = 0; i < k; i++) { - List tmp = new LinkedList(); - tmp.add(shufflelist.remove(0)); - - for (int j = 1; j < workingdata.size() / k ; j++) { - int nxt = r.nextInt(shufflelist.size()); - tmp.add(shufflelist.remove(nxt)); - } - this.clusters.add(tmp); - } - - - cluster(maxiters, swaps, n, workingdata, clusters); - - nullset = false; - - for (List cluster : clusters) { - nullset |= (cluster.size() == 0); - } - - } while (nullset && ++maxout<100); - if (maxout == 100) - System.err.println("Warning: MaxIterations Reached Outer"); - - } - - public void cluster(int maxiters, int swaps, int n, - ArrayList workingdata, List> clusters) { - while (swaps > 0 && maxiters > 0) { - maxiters--; - updateMeans(workingdata); - swaps = assignClusters(workingdata); - } - if (maxiters == 0) - System.err.println("Warning: MaxIterations Reached"); - updateMeans(this.data); - } - */ - - public Kmeans() { - // TODO Auto-generated constructor stub - } - - @Override - public List getCentroids() { - // if (means == null) { - // run(); - -// Rengine re = Rengine.getMainEngine(); -// if(re == null) -// re = new Rengine(new String[] {"--no-save"}, false, null); - -// if (!re.waitForR()) -// System.out.println("Cannot load R"); - - ArrayList workingdata = new ArrayList(); - for (float[] v : this.data) - workingdata.add(v); - List kmeansCentroids = new ArrayList(); - - // Convert List data to a 2D array - float[][] matrix = new float[workingdata.size()][]; - matrix = workingdata.toArray(matrix); - - // Get the number of rows and columns of the 2D array - int rows = matrix.length; - String numRows = String.valueOf(rows); - - int cols = matrix[0].length; - String numCols = String.valueOf(cols); - - // Set k - String kAsString = String.valueOf(k); - - // Convert the 2D array to a 1D double array to feed into R - double[] oneDArray = flatten(matrix); - -// // Feed the 1D array, k and number of rows and columns to R -// re.assign("data", oneDArray); -// re.assign("numberOfRows", numRows); -// re.assign("numberOfCols", numCols); -// re.assign("k", kAsString); -// -// // Create the data matrix in R -// re.eval("dataMatrix <- matrix(data, nrow = as.numeric(numberOfRows), ncol = as.numeric(numberOfCols), byrow = TRUE)"); -// -// // Run k-means in R -// double[][] kmOut = re.eval("kmeans(dataMatrix, as.numeric(k), nstart = 25)$centers").asDoubleMatrix(); - - // Convert the 2D array back to List format -// for (int i = 0; i < kmOut.length; i++) { -// float[] vector = new float[kmOut[0].length]; -// for (int j = 0; j < kmOut[0].length; j++) -// vector[j] = (float) kmOut[i][j]; -// kmeansCentroids.add(vector); -// } -// re.end(); - // } - List l = new ArrayList<>(); - for(float[] f : kmeansCentroids) - l.add(new Centroid(f,0)); - return l; - } - - // Convert a 2D array to a 1D double array - public static double[] flatten(float[][] twoDArray) { - ArrayList oneDArray = new ArrayList(); - - for (int i = 0; i < twoDArray.length; i++) - for (int j = 0; j < twoDArray[i].length; j++) - oneDArray.add((double) twoDArray[i][j]); - - Double[] doubles = oneDArray.toArray(new Double[0]); - double[] d = ArrayUtils.toPrimitive(doubles); - - return d; - } - - @Override - public void reset(int randomseed) { - - } - - public static void main(String[] args) { - GenerateData gen = new GenerateData(8, 100, 100); - Kmeans kk = new Kmeans(5, gen.data(), 24); -// VectorUtil.prettyPrint(kk.getCentroids()); - } - - @Override - public RPHashObject getParam() { - return new SimpleArrayReader(this.data, k); - } - - @Override - public boolean setMultiRun(int runs) { - return false; - } - -} diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/LloydIterativeKmeans.java b/src/main/java/edu/uc/rphash/tests/clusterers/LloydIterativeKmeans.java deleted file mode 100644 index 23ecf97..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/LloydIterativeKmeans.java +++ /dev/null @@ -1,250 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.Clusterer; -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.Readers.SimpleArrayReader; -import edu.uc.rphash.projections.DBFriendlyProjection; -import edu.uc.rphash.projections.Projector; -import edu.uc.rphash.tests.generators.GenerateData; -import edu.uc.rphash.util.VectorUtil; - -public class LloydIterativeKmeans implements Clusterer { - int k; - int n; - List data; - public int getK() { - return k; - } - - public void setK(int k) { - this.k = k; - } - - public List getData() { - return data; - } - - - @Override - public void setRawData(List data) { - this.data = data; - } - - @Override - public void setData(List centroids) { - ArrayList data = new ArrayList(centroids.size()); - for(Centroid c : centroids)data.add(c.centroid()); - setRawData(data); - } - - public List getWeights() { - return weights; - } - - public void setWeights(List weights) { - this.weights = weights; - } - - int projdim; - - List means; - List> clusters; - List weights; - - public LloydIterativeKmeans(int k, List data) { - this.k = k; - this.data = data; - this.projdim = 0; - this.clusters = null; - this.weights = new ArrayList(data.size()); - for (int i = 0; i < data.size(); i++) - weights.add(1f); - means = null; - } - - public LloydIterativeKmeans(int k, List data, List weights) { - this.k = k; - this.data = data; - this.projdim = 0; - this.clusters = null; - this.weights = weights; - means = null; - } - - public LloydIterativeKmeans(int k, List data, int projdim) { - this.k = k; - this.data = data; - this.projdim = projdim; - this.clusters = null; - this.weights = new ArrayList(data.size()); - for (int i = 0; i < data.size(); i++) - weights.add(1f); - means = null; - } - - public LloydIterativeKmeans() { - // TODO Auto-generated constructor stub - } - - public float[] computeCentroid(List vectors, List data) { - int d = data.get(0).length; - float[] centroid = new float[d]; - - for (int i = 0; i < d; i++) - centroid[i] = 0.0f; - - float w_total = 0f; - for (Integer v : vectors) { - w_total += weights.get(v); - } - - for (Integer v : vectors) { - float[] vec = data.get(v); - float weight = (float) weights.get(v) / (float) w_total; - for (int i = 0; i < d; i++) - centroid[i] += (vec[i] * weight); - } - return centroid; - } - - ArrayList weightTotals; - - void updateMeans(List data) { - weightTotals = new ArrayList(); - if (means == null) { - means = new ArrayList(); - for (int i = 0; i < k; i++) - means.add(computeCentroid(clusters.get(i), data)); - } - for (int i = 0; i < k; i++) - means.set(i, computeCentroid(clusters.get(i), data)); - } - - int assignClusters(List data) { - int swaps = 0; - List> newClusters = new ArrayList>(); - for (int j = 0; j < k; j++) - newClusters.add(new ArrayList()); - - for (int clusterid = 0; clusterid < k; clusterid++) { - - for (Integer member : clusters.get(clusterid)) { - - int nearest = VectorUtil.findNearestDistance(data.get(member), - means); - newClusters.get(nearest).add(member); - if (nearest != clusterid) - swaps++; - } - - } - clusters = newClusters; - return swaps; - } - - private void run() { - int maxiters = 1000; - int swaps = 2; - this.n = this.data.size(); - ArrayList workingdata = new ArrayList(); - // stuff for projected kmeans - Projector p = null; - Random r = new Random(); - if (projdim != 0) - p = new DBFriendlyProjection(this.data.get(0).length, projdim, - r.nextInt()); - for (float[] v : this.data) { - if (p != null) { - workingdata.add(p.project(v)); - } else - workingdata.add(v); - } - - int maxout = 0; - //loop until there are no more nullsets - boolean nullset = false; - do { - this.clusters = new ArrayList>(k); - // seed data with new clusters - ArrayList shufflelist = new ArrayList(data.size()); - for (int i = 0; i < data.size(); i++) - shufflelist.add(i); - - for (int i = 0; i < k; i++) { - List tmp = new LinkedList(); - tmp.add(shufflelist.remove(0)); - - for (int j = 1; j < workingdata.size() / k ; j++) { - int nxt = r.nextInt(shufflelist.size()); - tmp.add(shufflelist.remove(nxt)); - } - this.clusters.add(tmp); - } - - - cluster(maxiters, swaps, n, workingdata, clusters); - - nullset = false; - - for (List cluster : clusters) { - nullset |= (cluster.size() == 0); - } - - } while (nullset && ++maxout<100); - if (maxout == 100) - System.err.println("Warning: MaxIterations Reached Outer"); - - } - - public void cluster(int maxiters, int swaps, int n, - ArrayList workingdata, List> clusters) { - while (swaps > 0 && maxiters > 0) { - maxiters--; - updateMeans(workingdata); - swaps = assignClusters(workingdata); - } - if (maxiters == 0) - System.err.println("Warning: MaxIterations Reached"); - updateMeans(this.data); - } - - @Override - public List getCentroids() { - if (means == null) - run(); - List centroids = new ArrayList<>(means.size()); - for(float[] f : means){ - centroids.add(new Centroid(f,0)); - } - return centroids; - } - - @Override - public void reset(int randomseed) { - means = null; - } - - public static void main(String[] args) { - GenerateData gen = new GenerateData(8, 100, 100); - LloydIterativeKmeans kk = new LloydIterativeKmeans(5, gen.data(), 24); -// VectorUtil.prettyPrint(kk.getCentroids()); - } - - @Override - public RPHashObject getParam() { - - return new SimpleArrayReader(this.data, k); - } - - @Override - public boolean setMultiRun(int runs) { - return false; - } - -} diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/MLE2.java b/src/main/java/edu/uc/rphash/tests/clusterers/MLE2.java deleted file mode 100644 index 4519987..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/MLE2.java +++ /dev/null @@ -1,333 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.Clusterer; -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.Readers.SimpleArrayReader; - -/** - * @author lee - * learns mle model with T topics from words x docs counts data - */ -/* - * All private methods are so because they for speed reasons do not employ any - * form of checking for numerical stability. In the case of mle matrices this is - * acceptable as probability matrices are never negative and the dimensions of - * the matrices do not change. - * for a unit test of mle, mle will be used to produce the NMF of a matrix, functionality - * and correctness can be confirmed by finding the product to be equal to the input - * matrix - */ -public class MLE2 implements Clusterer { - - - - /** - * @param args - */ - public static void main(String[] args) { - float[][] F = {{1f, 0f, 0f, 2f, 0f},{1f, 1f, 1f, 0f, 0f}, {0f, 0f, 1f, 1f , 1f},{2f, 1f, 1f, 0f, 1f},{1f, 0f, 0f, 2f, 0f},{1f, 1f, 1f, 0f, 0f}, {0f, 0f, 1f, 1f , 1f},{2f, 1f, 1f, 0f, 1f}}; - for(float[] ff: F){ - System.out.println(); - for(float f: ff){ - System.out.print(f + " "); - } - }System.out.println(); - - - MLE2 mlobj = new MLE2(Arrays.asList(F),4,.00001f); - - printmat(normalize(F)); - printmat(mlobj.wt);printmat(mlobj.td); - printmat( multiply(mlobj.wt,mlobj.td)); - - } - - - int W;//words, rows - int D;//documents, columns - int T;// topics or latent classes - - public float[][] td; - public float[][] wt; - Listdata; - - public MLE2(List counts, int T,float epsilon) - { - this.data = counts; - W=counts.size(); - D = counts.get(0).length; - this.T = T; - mle(epsilon); - } - - // use if you want wt initialized to some specific value - public void mle( float epsilon) - { - float tot = sum(data); - td = normalize(ones(T,D)); - wt = normalize(rand(W,T)); - - - float[] E = sum1D(logDotProduct(data,multiply(wt,td))); - float F = sum(E)/tot; - float F_new ; - float rel_ch; - - - do - { - // Expectation Step - // td = norm(td .* ( wt' * ( counts ./ (wt * td) ) )); - td = normalize(dotProduct(td,(multiply(transpose(wt),dotDivide(data,multiply(wt,td)))))); - - //maximization step - //wt = normalize( wt .* ( ( counts ./ ( wt * td + eps ) ) * td' )) - wt = normalize(dotProduct(wt,multiply(dotDivide(data,multiply(wt,td)),transpose(td)))); - - //calculate log-likelihood - /* - * ___ ___ - * \ \ - * /__ /__ n(d,w) log P(d,w) - * d c D w c W - */ - E = sum1D(logDotProduct(data,multiply(wt,td))); - F_new = sum(E)/tot; - - //calculate iteration's relative change to determine convergence - rel_ch = Math.abs((F_new - F))/ Math.abs(F); - F= F_new; - - System.out.println(rel_ch); - - }while(rel_ch>epsilon); - - } - - - //testing status - works - //gets the pairwise products of two matrices - //no dimension checking - private static float[][] dotProduct(float[][] mat1, float[][] mat2) - { - float[][] rtrn = new float [mat1.length ][mat1[0].length]; - - for(int i = 0;i mat1, float[][] mat2) - { - float[][] rtrn = new float [mat1.size() ][mat1.get(0).length]; - - for(int i = 0;i mat1, float[][] mat2) - { - float[][] rtrn = new float [mat1.size() ][mat1.get(0).length]; - - for(int i = 0;i A) - { - float sum =0; - for(float[] ff:A){ - for(float f:ff) sum+=f; - } - return sum; - } - - //testing status - works - //give the sum of all the elements of a vector - public static float sum(float[]A) - { - float sum =0; - for(float f:A) sum+=f; - return sum; - } - - //testing status - works - //give the column vector sum of all the elements of a matrix - public static float[] sum1D(float[][] A) - { - float[] sum = new float[A[0].length]; - Arrays.fill(sum, 0f); - for(float[] ff:A) - { - for(int i = 0;i getCentroids() { - - // TODO Auto-generated method stub - return null; - } - - @Override - public RPHashObject getParam() { - return new SimpleArrayReader(this.data, T); - } - - @Override - public void setWeights(List counts) { - - } - - @Override - public void setRawData(List data) { - this.data = data; - } - - @Override - public void setData(List centroids) { - ArrayList data = new ArrayList(centroids.size()); - for(Centroid c : centroids)data.add(c.centroid()); - setRawData(data); - } - - @Override - public void setK(int getk) { - } - - @Override - public void reset(int randomseed) { - - } - - @Override - public boolean setMultiRun(int runs) { - return false; - } - -} diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/MaxLikelihoodKMeans2.java b/src/main/java/edu/uc/rphash/tests/clusterers/MaxLikelihoodKMeans2.java deleted file mode 100644 index e3f72fa..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/MaxLikelihoodKMeans2.java +++ /dev/null @@ -1,451 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Vector; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.Clusterer; -import edu.uc.rphash.RPHashStream; -import edu.uc.rphash.Readers.RPHashObject; -import edu.uc.rphash.tests.StatTests; -import edu.uc.rphash.tests.generators.GenerateStreamData; - -public class MaxLikelihoodKMeans2 implements Clusterer { - - class PointND { - - private int dimension; // number of coordinates of a point - private float[] coordinates; // the coordinates of a point - - /** - * Create a point centered at the origin of the specific dimension - **/ - public PointND(int dimension) { - this.dimension = dimension; - coordinates = new float[dimension]; - } - - public PointND(float[] data) { - this.dimension = data.length; - coordinates = data; - } - - /** - * Create a new point identical to point p - **/ - public PointND(PointND p) { - this.dimension = p.dimension; - this.coordinates = new float[dimension]; - for (int i = 0; i < dimension; i++) - this.coordinates[i] = p.coordinates[i]; - } - } - - private int n; // number of instances to classify - private int d; // number of coordinates of each point - private int k; // number of clusters - private PointND[] mu; // coordinate of means mu[j] of each cluster j - private Vector[] w; // holds the points classified into each class - // w[j] - private PointND[] sigma; // holds the standard deviation of each class i - private float[] prior; // holds the prior of each class i - // private float logLikelihood; // holds the log likelihood of each of the k - // Gaussians - private float MDL; // the minimum description length of the model - private int numIterations; - - private List centroids; - private PointND[] data; - - public MaxLikelihoodKMeans2(int getk, List data) { - this.data = new PointND[data.size()]; - for (int i = 0; i < data.size(); i++) { - this.data[i] = new PointND(data.get(i)); - } - this.centroids = null; - init(this.data, getk); - } - - public MaxLikelihoodKMeans2() { - } - - /** - * Intialize the parameters of the k-means algorithm Randomly assign a point - * in x to each mean mu[j] - **/ - private void init(PointND[] x, int k) { - this.n = x.length; - this.d = x[0].dimension; - this.k = k; - this.mu = new PointND[k]; - this.w = new Vector[k]; - this.numIterations = 0; - this.sigma = new PointND[k]; - this.prior = new float[k]; - - // randomly assign a point in x to each mean mu[j] - PointND randomPoint; - for (int j = 0; j < k; j++) { - randomPoint = x[(int) (Math.random() * (n - 1))]; - mu[j] = new PointND(randomPoint); - // each prior and standard deviation are set to zero - sigma[j] = new PointND(d); - prior[j] = 0; - } - } - - /** - * Runs the k-means algorithm with k clusters on the set of instances x Then - * find the quality of the model - **/ - public void run(PointND[] x, int k, float epsilon) { - float maxDeltaMeans = epsilon + 1; - PointND[] oldMeans = new PointND[k]; - // initialize n,k,mu[j] - init(x, k); - // iterate until there is no change in mu[j] - while (maxDeltaMeans > epsilon) { - // remember old values of the each mean - for (int j = 0; j < k; j++) { - oldMeans[j] = new PointND(mu[j]); - - } - - // classify each instance x[i] to its nearest class - // first we need to clear the class array since we are reclassifying - for (int j = 0; j < k; j++) { - w[j] = new Vector(); // could use clear but then have - // to init... - } - - for (int i = 0; i < n; i++) { - classify(x[i]); - } - // recompute each mean - computeMeans(); - // compute the largest change in mu[j] - maxDeltaMeans = maxDeltaMeans(oldMeans); - numIterations++; - } - } - - /** - * Classifies the point x to the nearest class - **/ - private void classify(PointND x) { - float dist = 0; - float smallestDist; - int nearestClass; - - // compute the distance x is from mean mu[0] - smallestDist = distance(x.coordinates, mu[0].coordinates); - nearestClass = 0; - - // compute the distance x is from the other classes - for (int j = 1; j < k; j++) { - dist = distance(x.coordinates, mu[j].coordinates); - if (dist < smallestDist) { - smallestDist = dist; - nearestClass = j; - } - } - // classify x into class its nearest class - w[nearestClass].add(x); - } - - float distance(float[] x, float[] y) { - float ret = 0.0f; - if (x.length != y.length) - return Float.MAX_VALUE; - for (int i = 0; i < x.length; i++) - ret += (x[i] - y[i]) * (x[i] - y[i]); - return (float) Math.sqrt(ret); - } - - float[] subtract(float[] x, float[] y) { - float[] ret = new float[x.length]; - if (x.length != y.length) - return null; - for (int i = 0; i < x.length; i++) - ret[i] = x[i] - y[i]; - return ret; - } - - float[] add(float[] x, float[] y) { - float[] ret = new float[x.length]; - if (x.length != y.length) - return null; - for (int i = 0; i < x.length; i++) - ret[i] = x[i] + y[i]; - return ret; - } - - float[] multiply(float[] x, float scalar) { - float[] ret = new float[x.length]; - for (int i = 0; i < x.length; i++) - ret[i] = x[i] * scalar; - return ret; - } - - public float max(float[] coordinates) { - float value; - float max = coordinates[0]; - for (int i = 1; i < coordinates.length; i++) { - value = coordinates[i]; - if (value > max) - max = value; - } - return max; - } - - /** - * Recompute mu[j] as the average of all points classified to the class w[j] - **/ - private void computeMeans() { - int numInstances; // number of instances in each class w[j] - PointND instance; - - // init the means to zero - for (int j = 0; j < k; j++) - mu[j] = new PointND(mu[j].dimension); - - // recompute the means of each cluster - for (int j = 0; j < k; j++) { - numInstances = w[j].size(); - for (int i = 0; i < numInstances; i++) { - instance = w[j].get(i); - mu[j] = new PointND( - add(mu[j].coordinates, instance.coordinates)); - // mu[j].add(instance); - } - // mu[j].multiply(1.0f / numInstances); - mu[j] = new PointND( - multiply(mu[j].coordinates, 1.0f / numInstances)); - } - - } - - /** - * Compute the maximum change over each mean mu[j] - **/ - private float maxDeltaMeans(PointND[] oldMeans) { - float delta; - oldMeans[0] = new PointND(subtract(oldMeans[0].coordinates, - mu[0].coordinates)); - // oldMeans[0].subtract(mu[0]); - - float maxDelta = max(oldMeans[0].coordinates); - for (int j = 1; j < k; j++) { - // oldMeans[j].subtract(mu[j]); - oldMeans[j] = new PointND(subtract(oldMeans[j].coordinates, - mu[j].coordinates)); - delta = max(oldMeans[j].coordinates); - if (delta > maxDelta) - maxDelta = delta; - } - return maxDelta; - } - - // /** - // * Compute the standard deviation of the k Gaussians - // **/ - // private void computeDeviation() { - // int numInstances; // number of instances in each class w[j] - // PointND instance; - // PointND temp; - // - // // set the standard deviation to zero - // for (int j = 0; j < k; j++) - // sigma[j].setToOrigin(); - // - // // for each cluster j... - // for (int j = 0; j < k; j++) { - // numInstances = w[j].size(); - // for (int i = 0; i < numInstances; i++) { - // instance = (PointND) (w[j].get(i)); - // temp = new PointND(instance); - // temp.subtract(mu[j]); - // temp.pow(2.0f); // (x[i]-mu[j])^2 - // temp.multiply(1.0f / numInstances); // multiply by proba of - // // having x[i] in cluster j - // sigma[j].add(temp); // sum i (x[i]-mu[j])^2 * p(x[i]) - // } - // sigma[j].pow((1.0f / 2f)); // because we want the standard deviation - // } - // } - // - // /** - // * Compute the priors of the k Gaussians - // **/ - // private void computePriors() { - // float numInstances; // number of instances in each class w[j] - // for (int j = 0; j < k; j++) { - // numInstances = w[j].size() * (1.0f); - // prior[j] = numInstances / n; - // } - // } - // - // /** - // * Assume the standard deviations and priors of each cluster have been - // * computed - // **/ - // private void computeLogLikelihood(PointND[] x) { - // float temp1 = 0; - // float temp2 = 0; - // PointND variance; - // float ln2 = (float) Math.log(2); - // // for each instance x - // for (int i = 0; i < n; i++) { - // // for each cluster j - // temp1 = 0; - // for (int j = 0; j < k; j++) { - // temp1 = temp1 + (x[i].normal(mu[j], sigma[j]) * prior[j]); - // } - // temp2 = (float) (temp2 + Math.log(temp1) / ln2); - // } - // logLikelihood = temp2; - // } - // - // /** - // * Assume the log likelihood and priors have been computed - // **/ - // private void computeMDL() { - // float temp = 0; - // float numInstances; - // float ln2 = (float) Math.log(2); - // for (int j = 0; j < k; j++) { - // numInstances = w[j].size(); - // for (int i = 0; i < d; i++) { - // temp = (float) (temp - Math.log(sigma[j].getCoordinate(i) - // / Math.sqrt(numInstances)) - // / ln2); - // } - // } - // MDL = temp - logLikelihood; - // } - - public float getMDL() { - return MDL; - } - - public List getCentroids() { - float epsilon = 0.01f; - if (centroids != null){ - - return centroids; - } - init(data, k); - run(data, d, epsilon); - centroids = new ArrayList(k); - for (int i = 0; i < k; i++) - centroids.add(new Centroid(mu[i].coordinates,0)); - - // compute sum of squares - double sigtotal = 0.0; - for (int i = 0; i < sigma.length; i++) - for (int j = 0; j < sigma[i].dimension; j++) - sigtotal += sigma[i].coordinates[j]; - - return centroids; - } - - @Override - public RPHashObject getParam() { - // TODO Auto-generated method stub - return null; - } - - @Override - public void setWeights(List counts) { - - } - - @Override - public void setData(List data) { - this.centroids = null; - this.data = new PointND[data.size()]; - for (int i = 0; i < data.size(); i++) { - this.data[i] = new PointND(data.get(i).centroid()); - } - } - @Override - public void setRawData(List data) { - this.centroids = null; - this.data = new PointND[data.size()]; - for (int i = 0; i < data.size(); i++) { - this.data[i] = new PointND(data.get(i)); - } - } - - @Override - public void setK(int getk) { - this.k = getk; - } - - @Override - public void reset(int randomseed) { - centroids = null; - } - - public static void main(String[] args) { - int k = 10; - int d = 240; - float var = 1f; - int interval = 1000; - Runtime rt = Runtime.getRuntime(); - - GenerateStreamData gen1 = new GenerateStreamData(k, d, var, 11331313); - GenerateStreamData noise = new GenerateStreamData(1, d, var * 10, - 11331313); - MaxLikelihoodKMeans2 km2 = new MaxLikelihoodKMeans2(); - // HartiganWongKMeans hwkm = new HartiganWongKMeans(); - - System.out.printf("\tKMeans\t\t\tNull\t\tReal\n"); - System.out - .printf("Vecs\tMem(KB)\tTime\tWCSSE\t\tTime\tWCSSE\t\tWCSSE\n"); - - long timestart = System.nanoTime(); - for (int i = 0; i < 2500000;) { - ArrayList vecsAndNoiseInThisRound = new ArrayList(); - ArrayList justvecsInThisRound = new ArrayList(); - - for (int j = 1; j < interval && i < 2500000; i++, j++) { - float[] vec = gen1.generateNext(); - vecsAndNoiseInThisRound.add(vec); - justvecsInThisRound.add(vec); - vecsAndNoiseInThisRound.add(noise.generateNext()); - } - - timestart = System.nanoTime(); - km2.setRawData(vecsAndNoiseInThisRound); - km2.setK(k); - - List cents = km2.getCentroids(); - long time = System.nanoTime() - timestart; - - rt.gc(); - long usedkB = (rt.totalMemory() - rt.freeMemory()) / 1024; - - double wcsse = StatTests.WCSSECentroidsFloat(cents, justvecsInThisRound); - double realwcsse = StatTests.WCSSE(gen1.medoids, - justvecsInThisRound); - System.out.printf("%d\t%d\t%.4f\t%.1f\t\t", i, usedkB, - time / 1000000000f, wcsse); - - cents = new HartiganWongKMeans(k, vecsAndNoiseInThisRound) - .getCentroids(); - time = System.nanoTime() - timestart; - usedkB = (rt.totalMemory() - rt.freeMemory()) / 1024; - wcsse = StatTests.WCSSECentroidsFloat(cents, justvecsInThisRound); - System.out.printf("%.4f\t%.1f\t\t%.1f\n", time / 1000000000f, - wcsse, realwcsse); - } - } - @Override - public boolean setMultiRun(int runs) { - return false; - } - -} diff --git a/src/main/java/edu/uc/rphash/tests/clusterers/SVD.java b/src/main/java/edu/uc/rphash/tests/clusterers/SVD.java deleted file mode 100644 index da67542..0000000 --- a/src/main/java/edu/uc/rphash/tests/clusterers/SVD.java +++ /dev/null @@ -1,456 +0,0 @@ -package edu.uc.rphash.tests.clusterers; - -import java.util.ArrayList; -import java.util.Collections; - -public class SVD { - /** - * returns U in a. normaly U is nr*nr, but if nr>nc only the first nc - * columns are returned (nice, saves memory). The columns of U have - * arbitrary sign, also the columns corresponding to near-zero singular - * values can vary wildly from other implementations. - *This function is adapted from the c coded method from Numerical Recipes in C - */ - - private float[][] A; - SVDMatrix svdmat; - - public SVD(float[][] A) - { - this.A=A; - } - - public void compute() - { - - - - float[] D = new float[A[0].length]; - float[][] V = new float[A[0].length][A[0].length]; - - svdmat = new SVDMatrix(A,D,V,A.length < A[0].length); - - svd(svdmat.getU(),svdmat.getD(),svdmat.getV()); - } - - public float[][] getU(){ - return svdmat.getV(); - } - - public float[][] getD() - { - return padV(svdmat.getD()); - } - - public float[][] getVT(){ - return transpose(svdmat.getV()); - } - - public void svd(float[][] a, float[] w, float[][] v) { - int i, its, j, jj, k, l = 0, nm = 0; - boolean flag; - int m = a.length; - int n = a[0].length; - float c, f, h, s, x, y, z; - float anorm = 0.f, g = 0.f, scale = 0.f; - float[] rv1 = new float[n]; - - for (i = 0; i < n; i++) { - l = i + 1; - rv1[i] = scale * g; - g = s = scale = 0.f; - if (i < m) { - for (k = i; k < m; k++) - scale += abs(a[k][i]); - if (scale != 0.0) { - for (k = i; k < m; k++) { - a[k][i] /= scale; - s += a[k][i] * a[k][i]; - } - f = a[i][i]; - g = -SIGN((float)Math.sqrt(s), f); - h = f * g - s; - a[i][i] = f - g; - // if (i!=(n-1)) { // CHECK - for (j = l; j < n; j++) { - for (s = 0, k = i; k < m; k++) - s += a[k][i] * a[k][j]; - f = s / h; - for (k = i; k < m; k++) - a[k][j] += f * a[k][i]; - } - // } - for (k = i; k < m; k++) - a[k][i] *= scale; - } - } - w[i] = scale * g; - g = s = scale = 0.0f; - if (i < m && i != n - 1) { // - for (k = l; k < n; k++) - scale += abs(a[i][k]); - if (scale != 0.) { - for (k = l; k < n; k++) { // - a[i][k] /= scale; - s += a[i][k] * a[i][k]; - } - f = a[i][l]; - g = -SIGN((float)Math.sqrt(s), f); - h = f * g - s; - a[i][l] = f - g; - for (k = l; k < n; k++) - rv1[k] = a[i][k] / h; - if (i != m - 1) { // - for (j = l; j < m; j++) { // - for (s = 0, k = l; k < n; k++) - s += a[j][k] * a[i][k]; - for (k = l; k < n; k++) - a[j][k] += s * rv1[k]; - } - } - for (k = l; k < n; k++) - a[i][k] *= scale; - } - } // i= 0; --i) { - if (i < n - 1) { // - if (g != 0.) { - for (j = l; j < n; j++) - v[j][i] = (a[i][j] / a[i][l]) / g; - for (j = l; j < n; j++) { - for (s = 0, k = l; k < n; k++) - s += a[i][k] * v[k][j]; - for (k = l; k < n; k++) - v[k][j] += s * v[k][i]; - } - } - for (j = l; j < n; j++) - // - v[i][j] = v[j][i] = 0.0f; - } - v[i][i] = 1.0f; - g = rv1[i]; - l = i; - } - // for (i=IMIN(m,n);i>=1;i--) { // ! - // for (i = n-1; i>=0; --i) { - for (i = Math.min(m - 1, n - 1); i >= 0; --i) { - l = i + 1; - g = w[i]; - if (i < n - 1) // - for (j = l; j < n; j++) - // - a[i][j] = 0.0f; - if (g != 0.) { - g = 1.f / g; - if (i != n - 1) { - for (j = l; j < n; j++) { - for (s = 0, k = l; k < m; k++) - s += a[k][i] * a[k][j]; - f = (s / a[i][i]) * g; - for (k = i; k < m; k++) - a[k][j] += f * a[k][i]; - } - } - for (j = i; j < m; j++) - a[j][i] *= g; - } else { - for (j = i; j < m; j++) - a[j][i] = 0.0f; - } - a[i][i] += 1.0; - } - for (k = n - 1; k >= 0; --k) { - for (its = 1; its <= 30; ++its) { - flag = true; - for (l = k; l >= 0; --l) { - nm = l - 1; - if ((abs(rv1[l]) + anorm) == anorm) { - flag = false; - break; - } - if ((abs(w[nm]) + anorm) == anorm) - break; - } - if (flag) { - c = 0.0f; - s = 1.0f; - for (i = l; i <= k; i++) { // - f = s * rv1[i]; - rv1[i] = c * rv1[i]; - if ((abs(f) + anorm) == anorm) - break; - g = w[i]; - h = pythag(f, g); - w[i] = h; - h = 1.0f / h; - c = g * h; - s = -f * h; - for (j = 0; j < m; j++) { - y = a[j][nm]; - z = a[j][i]; - a[j][nm] = y * c + z * s; - a[j][i] = z * c - y * s; - } - } - } // flag - z = w[k]; - if (l == k) { - if (z < 0.) { - w[k] = -z; - for (j = 0; j < n; j++) - v[j][k] = -v[j][k]; - } - break; - } // l==k - x = w[l]; - nm = k - 1; - y = w[nm]; - g = rv1[nm]; - h = rv1[k]; - f = ((y - z) * (y + z) + (g - h) * (g + h)) / (2 * h * y); - g = pythag(f, 1.0f); - f = ((x - z) * (x + z) + h * ((y / (f + SIGN(g, f))) - h)) / x; - c = s = 1.0f; - for (j = l; j <= nm; j++) { - i = j + 1; - g = rv1[i]; - y = w[i]; - h = s * g; - g = c * g; - z = pythag(f, h); - rv1[j] = z; - c = f / z; - s = h / z; - f = x * c + g * s; - g = g * c - x * s; - h = y * s; - y *= c; - for (jj = 0; jj < n; jj++) { - x = v[jj][j]; - z = v[jj][i]; - v[jj][j] = x * c + z * s; - v[jj][i] = z * c - x * s; - } - z = pythag(f, h); - w[j] = z; - if (z != 0.0) { - z = 1.0f / z; - c = f * z; - s = h * z; - } - f = c * g + s * y; - x = c * y - s * g; - for (jj = 0; jj < m; ++jj) { - y = a[jj][j]; - z = a[jj][i]; - a[jj][j] = y * c + z * s; - a[jj][i] = z * c - y * s; - } - } // j= 0. ? abs(a) : -abs(a)); - } - - //creates a diagonal matrix by padding the vector(v) with zeros - public static float[][] padV(float[] v) - { - float rtrn[][] = new float[v.length][v.length]; - for(int i =0;i svdpairs; - float[] D; - float[][] U; - float[][]V; - boolean sorted; - - - public SVDMatrix(float[][] u, float[] d, float[][]v, boolean transpose) - { - transpose = false; - sorted = false; - - /* if(transpose){ - D=d; - U=transpose(v); - V=u; - }else - { - */ D=d; - U=u; - V=v; - // } - } - - public void sortSingularValues() - { - svdpairs = new ArrayList(D.length); - - for(int i = 0; i< D.length;i++){ - float urow[] = new float[U[0].length]; - float vrow[] = new float[V[0].length]; - - for(int j = 0; j { - float singularValue; - float[] urows; - float[] vrows; - - public SVDValuePairs(float singularValue, float[] urows,float[] vrows){ - this.singularValue = singularValue; - this.vrows = vrows; - this.urows = urows; - } - - public int compareTo(SVDValuePairs o) { - if(o instanceof SVDValuePairs) - return 0; - if(((SVDValuePairs)o).singularValue == this.singularValue)return 0; - if(((SVDValuePairs)o).singularValue > this.singularValue)return 1; - return -1; - } - } - } -} diff --git a/src/main/java/edu/uc/rphash/tests/testStreamingRPHash.java b/src/main/java/edu/uc/rphash/tests/testStreamingRPHash.java deleted file mode 100644 index ac62432..0000000 --- a/src/main/java/edu/uc/rphash/tests/testStreamingRPHash.java +++ /dev/null @@ -1,184 +0,0 @@ -package edu.uc.rphash.tests; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import edu.uc.rphash.Centroid; -import edu.uc.rphash.RPHashMultiProj; -import edu.uc.rphash.RPHashStream; -import edu.uc.rphash.tests.clusterers.StreamingKmeans; -import edu.uc.rphash.tests.clusterers.StreamingKmeans2; -import edu.uc.rphash.tests.generators.GenerateStreamData; -import edu.uc.rphash.util.VectorUtil; - -public class testStreamingRPHash { - public static void readFileData(String[] args) throws Exception { - - int interval = 1000; - int k = 10; - String filename = "/home/lee/Desktop/Dimension3204/data.mat"; - int processors = Runtime.getRuntime().availableProcessors(); - if (args.length > 1) - filename = args[0]; - if (args.length > 2) - k = Integer.parseInt(args[1]); - if (args.length > 3) - processors = Integer.parseInt(args[0]); - - Runtime rt = Runtime.getRuntime(); - List data = VectorUtil.readFile(filename, false); - - RPHashStream rphit = new RPHashStream(data, k); - - // System.out.printf("Running Streaming RPHash on %d processors, d=%d,k=%d,n=%d\n",rphit.getProcessors(),d,k,interval); - // StreamClusterer rphit = new StreamingKmeans(data, k); - // System.out.printf("Running Streaming KMeans on %d processors, d=%d,k=%d\n",1,data.size(),k); - - System.out.printf("Vecs\tMem(KB)\tTime\tWCSSE\n"); - long timestart = System.nanoTime(); - - timestart = System.nanoTime(); - rphit.addVectorOnlineStep(data.get(0)); - for (int i = 1; i < 20000; i++) { - rphit.addVectorOnlineStep(data.get(i)); - - if (i % interval == 0) { - List cents = rphit.getCentroidsOfflineStep(); - long time = System.nanoTime() - timestart; - - rt.gc(); - long usedkB = (rt.totalMemory() - rt.freeMemory()) / 1024; - - double wcsse = StatTests.WCSSECentroidsFloat(cents, data); - - System.gc(); - System.out.printf("%d\t%d\t%.4f\t%.0f\n", i, usedkB, - time / 1000000000f, wcsse); - timestart = System.nanoTime(); - } - } - - } - - public static void generateAndStream() throws InterruptedException { - Random r = new Random(); - int k = 10; - int d = 1000; - float var = 1f; - int interval = 10000; - Runtime rt = Runtime.getRuntime(); - - GenerateStreamData gen1 = new GenerateStreamData(k, d, var, 11331313); - GenerateStreamData noise = new GenerateStreamData(1, d, var*10, 11331313); - RPHashStream rphit = new RPHashStream(k, gen1,rt.availableProcessors()); - StreamingKmeans2 skmi = new StreamingKmeans2(k, gen1,rt.availableProcessors()); - System.out.printf("\tStreamingRPHash\t\t\tStreamingKmeans\t\tReal\n"); - System.out.printf("Vecs\tMem(KB)\tTime\tWCSSE\t\tTime\tWCSSE\t\tWCSSE\n"); - - long timestart = System.nanoTime(); - for (int i = 0; i < 2500000;) { - ArrayList vecsAndNoiseInThisRound = new ArrayList(); - ArrayList justvecsInThisRound = new ArrayList(); - - for (int j = 1; j < interval && i < 2500000; i++, j++){ - float[] vec = gen1.generateNext(); - vecsAndNoiseInThisRound.add(vec); - justvecsInThisRound.add(vec); - if(r.nextInt(10)==1) - vecsAndNoiseInThisRound.add(noise.generateNext()); - } - - timestart = System.nanoTime(); - for (float[] f : vecsAndNoiseInThisRound) { - rphit.addVectorOnlineStep(f); - } - List cents = rphit.getCentroidsOfflineStep(); - long time = System.nanoTime() - timestart; - - rt.gc(); - long usedkB = (rt.totalMemory() - rt.freeMemory()) / 1024; - - double wcsse = StatTests.WCSSECentroidsFloat(cents, justvecsInThisRound); - double realwcsse = StatTests.WCSSE(gen1.medoids, justvecsInThisRound); - - System.out.printf("%d\t%d\t%.4f\t%.1f\t\t", i, usedkB, - time / 1000000000f, wcsse); - rt.gc(); - Thread.sleep(1000); - rt.gc(); - - timestart = System.nanoTime(); - for (float[] f : vecsAndNoiseInThisRound) { - skmi.addVectorOnlineStep(f); - } - - cents = skmi.getCentroidsOfflineStep(); - time = System.nanoTime() - timestart; - - rt.gc(); - usedkB = (rt.totalMemory() - rt.freeMemory()) / 1024; - - wcsse = StatTests.WCSSECentroidsFloat(cents, justvecsInThisRound); - // recreate vectors at execution time to check average - rt.gc(); - Thread.sleep(1000); - rt.gc(); - - System.out.printf("%.4f\t%.1f\t\t%.1f\n",time/ 1000000000f,wcsse,realwcsse); - } - } - - public static void streamingPushtest() { - int k = 10; - int d = 1000; - float var = 4.5f; - - GenerateStreamData gen1 = new GenerateStreamData(k, d, var, 11331313); - - RPHashStream rphit = new RPHashStream(k,gen1); - - ArrayList cts = new ArrayList(); - for (int i = 0; i < 10000; i++) { - long centroidCount = rphit.addVectorOnlineStep(gen1.generateNext()); -// if (centroidCount>1 ) { -// cts.add((int) centroidCount); -// List f = rphit.getTopIdSizes(); -// for (float ff : f) -// System.out.print(ff/(float)i + ","); -// System.out.print("]\n["); -// } - } - //System.out.println(cts.toString()); - } - - public static void main(String[] args) throws Exception { -// readFileData(args); - generateAndStream(); -// streamingPushtest(); - } - - static void prettyPrint(List cs){ - - int n = cs.get(0).centroid.length; - boolean curtailm = n > 10; - if (curtailm) { - for (int i = 0; i < 4; i++) { - VectorUtil.prettyPrint(cs.get(i).centroid); - } - for (int j = 0; j < n / 2; j++) - System.out.print("\t"); - System.out.print(" ...\n"); - for (int i = cs.size() - 4; i < cs.size(); i++) { - VectorUtil.prettyPrint(cs.get(i).centroid); - } - } else { - for (int i = 0; i < cs.size(); i++) { - VectorUtil.prettyPrint(cs.get(i).centroid); - System.out.print("\n"); - } - } - - } - -} From 6fb8e771ba4602619856de7c96464165b635b079 Mon Sep 17 00:00:00 2001 From: deysn Date: Tue, 3 Jul 2018 03:26:25 -0400 Subject: [PATCH 2/8] Adding the options for TWRP from the main RPHASH class to run from the command line. --- src/main/java/edu/uc/rphash/RPHash.java | 8 +- .../edu/uc/rphash/RPHashAdaptive2Pass.java | 63 +++--- .../edu/uc/rphash/Readers/RPHashObject.java | 2 +- src/main/java/edu/uc/rphash/TWRP1.java | 27 +-- src/main/java/edu/uc/rphash/TWRPv2.java | 205 ++++++------------ .../projections/DBFriendlyProjection.java | 20 +- 6 files changed, 119 insertions(+), 206 deletions(-) diff --git a/src/main/java/edu/uc/rphash/RPHash.java b/src/main/java/edu/uc/rphash/RPHash.java index 2977bc4..9038026 100644 --- a/src/main/java/edu/uc/rphash/RPHash.java +++ b/src/main/java/edu/uc/rphash/RPHash.java @@ -51,7 +51,7 @@ public class RPHash { static String[] clusteringmethods = { "simple", "streaming", "multiproj", - "kmeans", "pkmeans","kmeansplusplus", "streamingkmeans", "adaptive","dummy" }; + "kmeans", "pkmeans","kmeansplusplus", "streamingkmeans", "adaptive","dummy" ,"twrp"}; static String[] offlineclusteringmethods = { "singlelink", "completelink", "averagelink", "kmeans", "adaptivemeanshift", "kmpp", "multikmpp" , "dbscan", "none" }; static String[] projectionmethods = { "dbf", "fjlt", "rp", "svd", "noproj" }; @@ -720,6 +720,12 @@ public static List runConfigs(List untaggedArgs, runitems.add(new RPHashAdaptive2Pass(o)); break; } + + case "twrp": { + runitems.add(new TWRPv2(o)); + break; + } + case "dummy": { runitems.add(new DummyClusterer(so)); break; diff --git a/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java b/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java index 1ff2008..88a9637 100644 --- a/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java +++ b/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java @@ -170,32 +170,22 @@ public List> findDensityModes() { } - for (Long name: IDAndCent.keySet()){ - - String key =name.toString(); - - - String value = IDAndCent.get(name).toString() ; - - // String value1 = Arrays.toString(value.toString()); - - System.out.println(key + " " + value); - - - - -} - - for (Long name: IDAndID.keySet()){ - - String key =name.toString(); - String value = IDAndID.get(name).toString(); - System.out.println(key + " " + value); - - -} - - +// for (Long name: IDAndCent.keySet()){ +// +// String key =name.toString(); +// // String value = IDAndCent.get(name).toString() ; +// // String value1 = Arrays.toString(value.toString()); +// System.out.println(key ) ;//+ " " + value); +//} + + System.out.println("NumberOfMicroClustersBeforePruning = "+ IDAndCent.size()); +// for (Long name: IDAndID.keySet()){ +// String key =name.toString(); +// String value = IDAndID.get(name).toString(); +// System.out.println(key + " " + value); +// +// +//} // next we want to prune the tree by parent count comparison // follows breadthfirst search @@ -249,6 +239,8 @@ public List> findDensityModes() { // System.out.println(sortedIDList.get(i) + ":"+VectorUtil.longToString(sortedIDList.get(i))+":"+IDAndCent.get(sortedIDList.get(i)).size()); // } +// System.out.println("NumberOfMicroClusters_AfterPruning = "+ denseSetOfIDandCount.size()); + System.out.println("NumberOfMicroClusters_AfterPruning = "+ estcents.size()); return new ArrayList<>(estcents.values()); @@ -271,7 +263,7 @@ public void run() { centroids.add(medoid(clustermembers.get(i))); } Agglomerative3 aggloOffline = new Agglomerative3(centroids, so.getk()); - System.out.println(centroids.size()); +// System.out.println(centroids.size()); aggloOffline.setWeights(weights); this.centroids = aggloOffline.getCentroids(); } @@ -279,17 +271,17 @@ public void run() { public static void main(String[] args) throws FileNotFoundException, IOException { - int k = 3; + int k = 6; int d = 100; - int n = 2000; - float var = 1.0f;//0.5f; + int n = 5000; + float var = 1.5f;//0.5f; int count = 1; // System.out.printf("ClusterVar\t"); // for (int i = 0; i < count; i++) // System.out.printf("Trial%d\t", i); // System.out.printf("RealWCSS\n"); - for (float f = var; f < 1.01; f += 1.5f) { + for (float f = var; f < 1.51; f += 1.5f) { float avgrealwcss = 0; float avgtime = 0; // System.out.printf("%f\t", f); @@ -298,7 +290,7 @@ public static void main(String[] args) throws FileNotFoundException, // gen.writeCSVToFile(new // File("/home/lee/Desktop/reclsh/in.csv")); RPHashObject o = new SimpleArrayReader(gen.data, k); - o.setDimparameter(4); + o.setDimparameter(8); RPHashAdaptive2Pass rphit = new RPHashAdaptive2Pass(o); long startTime = System.nanoTime(); List centsr = rphit.getCentroids(); @@ -312,11 +304,10 @@ public static void main(String[] args) throws FileNotFoundException, String Output = "/C:/Users/user/Desktop/temp/OutputTwrpCents" ; VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); - // System.out.printf("%.0f\t", - // StatTests.WCSSECentroidsFloat(centsr, gen.data)); - // System.gc(); + System.out.printf("%.0f\n\t",StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); } - // System.out.printf("%.0f\n", avgrealwcss / count); + System.out.printf("%.0f\n", avgrealwcss / count); diff --git a/src/main/java/edu/uc/rphash/Readers/RPHashObject.java b/src/main/java/edu/uc/rphash/Readers/RPHashObject.java index 9f38946..5ecb470 100644 --- a/src/main/java/edu/uc/rphash/Readers/RPHashObject.java +++ b/src/main/java/edu/uc/rphash/Readers/RPHashObject.java @@ -29,7 +29,7 @@ public interface RPHashObject { final static int DEFAULT_NUM_PROJECTIONS = 1; public final static int DEFAULT_NUM_BLUR = 1; - final static long DEFAULT_NUM_RANDOM_SEED = 38006359550206753L; + final static long DEFAULT_NUM_RANDOM_SEED = 3800635955020675334L; final static int DEFAULT_NUM_DECODER_MULTIPLIER = 1; final static long DEFAULT_HASH_MODULUS = Long.MAX_VALUE; final static Decoder DEFAULT_INNER_DECODER = new Spherical(32,4,1);//new DepthProbingLSH(24);//new Leech();//new Spherical(16,2,2);//new MultiDecoder(24, new E8(1f));//new Golay();//new Spherical(64,2,1);//new Leech(3);//new PsdLSH();// diff --git a/src/main/java/edu/uc/rphash/TWRP1.java b/src/main/java/edu/uc/rphash/TWRP1.java index aaf3540..7a9f923 100644 --- a/src/main/java/edu/uc/rphash/TWRP1.java +++ b/src/main/java/edu/uc/rphash/TWRP1.java @@ -369,13 +369,7 @@ public List> findDensityModes() { return new ArrayList<>(estcents.values()); } - - - - - - - + @@ -605,9 +599,7 @@ public HashMap findDensityModes2() { } - - - + public void run() { @@ -682,16 +674,16 @@ public static void main(String[] args) throws FileNotFoundException, IOException { int k = 6;//6; - int d = 64;//16; - int n = 700; - float var = .5f; + int d = 100;//16; + int n = 5000; + float var = 1.5f; int count = 1; // System.out.printf("ClusterVar\t"); // for (int i = 0; i < count; i++) // System.out.printf("Trial%d\t", i); // System.out.printf("RealWCSS\n"); - for (float f = var; f < 1.01; f += 1.5f) { + for (float f = var; f < 1.51; f += 1.5f) { float avgrealwcss = 0; float avgtime = 0; // System.out.printf("%f\t", f); @@ -700,7 +692,7 @@ public static void main(String[] args) throws FileNotFoundException, // gen.writeCSVToFile(new // File("/home/lee/Desktop/reclsh/in.csv")); RPHashObject o = new SimpleArrayReader(gen.data, k); - o.setDimparameter(4); + o.setDimparameter(8); TWRP1 rphit = new TWRP1(o); long startTime = System.nanoTime(); List centsr = rphit.getCentroids(); @@ -710,9 +702,8 @@ public static void main(String[] args) throws FileNotFoundException, avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), gen.getData()); - // System.out.printf("%.0f\t", - // StatTests.WCSSECentroidsFloat(centsr, gen.data)); - // System.gc(); + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); } System.out.printf("%.0f\n", avgrealwcss / count); diff --git a/src/main/java/edu/uc/rphash/TWRPv2.java b/src/main/java/edu/uc/rphash/TWRPv2.java index 08b05a2..d24a616 100644 --- a/src/main/java/edu/uc/rphash/TWRPv2.java +++ b/src/main/java/edu/uc/rphash/TWRPv2.java @@ -4,15 +4,15 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; +//import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; +//import java.util.Iterator; import java.util.List; import java.util.Map.Entry; import java.util.Random; import java.util.TreeSet; import java.util.stream.Stream; -import java.util.Map; +//import java.util.Map; import edu.uc.rphash.Readers.RPHashObject; @@ -24,17 +24,14 @@ import edu.uc.rphash.util.VectorUtil; - public class TWRPv2 implements Clusterer, Runnable { boolean znorm = false; - private int counter; private float[] rngvec; private List centroids = null; - private RPHashObject so; public TWRPv2(RPHashObject so) { @@ -89,43 +86,39 @@ public static float[][] UpdateHashMap(float cnt_1, float[] x_1, ret[1] = x_r; return ret; } - - - - - + //float[] rngvec; the range vector is moot if incoming data has been normalized //post normalization it should all be zero centered, with variance 1 - + /* * super simple hash algorithm, reminiscient of pstable lsh */ // xt is the projected vector and x is the original vector , rngvec is the randomly generated vector of projected dim. - public long hashvec(float[] xt, float[] x, - HashMap> IDAndCent, HashMap> IDAndLabel,int ct) { - long s = 1; //fixes leading 0's bug - for (int i = 0; i < xt.length; i++) { -// s <<= 1; - s = s << 1 ; // left shift the bits of s by 1. - if (xt[i] > rngvec[i]) -// s += 1; - s= s+1; - - if (IDAndCent.containsKey(s)) { - IDAndLabel.get(s).add(ct); - IDAndCent.get(s).add(x); - } else { - List xlist = new ArrayList<>(); - xlist.add(x); - IDAndCent.put(s, xlist); - List idlist = new ArrayList<>(); - idlist.add(ct); - IDAndLabel.put(s, idlist); - } - } - return s; - } +// public long hashvec(float[] xt, float[] x, +// HashMap> IDAndCent, HashMap> IDAndLabel,int ct) { +// long s = 1; //fixes leading 0's bug +// for (int i = 0; i < xt.length; i++) { +//// s <<= 1; +// s = s << 1 ; // left shift the bits of s by 1. +// if (xt[i] > rngvec[i]) +//// s += 1; +// s= s+1; +// +// if (IDAndCent.containsKey(s)) { +// IDAndLabel.get(s).add(ct); +// IDAndCent.get(s).add(x); +// } else { +// List xlist = new ArrayList<>(); +// xlist.add(x); +// IDAndCent.put(s, xlist); +// List idlist = new ArrayList<>(); +// idlist.add(ct); +// IDAndLabel.put(s, idlist); +// } +// } +// return s; +// } public long hashvec2(float[] xt, float[] x, HashMap MapOfIDAndCent, HashMap MapOfIDAndCount,int ct) { @@ -136,9 +129,7 @@ public long hashvec2(float[] xt, float[] x, if (xt[i] > rngvec[i]) // s += 1; s= s+1; - - - + if (MapOfIDAndCent.containsKey(s)) { float CurrentCount = MapOfIDAndCount.get(s); @@ -157,8 +148,7 @@ public long hashvec2(float[] xt, float[] x, MapOfIDAndCent.put(s, MergedVector); } - - + else { float[] xlist = x; @@ -178,7 +168,7 @@ public long hashvec2(float[] xt, float[] x, * maps */ void addtocounter(float[] x, Projector p, - HashMap> IDAndCent,HashMap> IDandID,int ct) { + HashMap IDAndCent,HashMap IDandID,int ct) { float[] xt = p.project(x); // counter++; @@ -187,11 +177,11 @@ void addtocounter(float[] x, Projector p, // rngvec[i] += delta/(float)counter; // } - hashvec(xt,x,IDAndCent, IDandID,ct); + hashvec2(xt,x,IDAndCent, IDandID,ct); } void addtocounter(float[] x, Projector p, - HashMap> IDAndCent,HashMap> IDandID,int ct,float[] mean,float[] variance) + HashMap IDAndCent,HashMap IDandID,int ct,float[] mean,float[] variance) { float[] xt = p.project(StatTests.znormvec(x, mean, variance)); @@ -201,7 +191,7 @@ void addtocounter(float[] x, Projector p, // rngvec[i] += delta/(float)counter; // } - hashvec(xt,x,IDAndCent, IDandID,ct); + hashvec2(xt,x,IDAndCent, IDandID,ct); } static boolean isPowerOfTwo(long num) { @@ -214,10 +204,9 @@ static boolean isPowerOfTwo(long num) { */ - public HashMap findDensityModes2() { - HashMap> IDAndCent = new HashMap<>(); - HashMap> IDAndID = new HashMap<>(); + HashMap MapOfIDAndCent = new HashMap<>(); + HashMap MapOfIDAndCount = new HashMap<>(); // #create projector matrixs Projector projector = so.getProjectionType(); projector.setOrigDim(so.getdim()); @@ -232,7 +221,7 @@ public HashMap findDensityModes2() { // // #process data by adding to the counter // for (float[] x : so.getRawData()) // { -// addtocounter(x, projector, IDAndCent,IDAndID,ct++,mean,variance); +// addtocounter(x, projector, MapOfIDAndCent,MapOfIDAndCount,ct++,mean,variance); // } // } // @@ -241,82 +230,34 @@ public HashMap findDensityModes2() { for (float[] x : so.getRawData()) { - addtocounter(x, projector, IDAndCent, IDAndID,ct++); + addtocounter(x, projector, MapOfIDAndCent, MapOfIDAndCount,ct++); } } - for (Long name: IDAndCent.keySet()){ +// for (Long name: MapOfIDAndCent.keySet()){ - String key =name.toString(); - System.out.println(key ); +// String key =name.toString(); +// System.out.println(key); // String value = IDAndCent.get(name).toString() ; -// String value1 = Arrays.toString(value.toString()); - +// String value1 = Arrays.toString(value.toString()); // System.out.println(key + " " + value); - -} +//} - for (Long name: IDAndID.keySet()){ +// for (Long name: MapOfIDAndCount.keySet()){ // String key =name.toString(); // String value = IDAndID.get(name).toString(); // System.out.println(key + " " + value); - -} - - // we would compress the hashmaps. SetOfIDandCount has the ids and the counts corresponding to that id. - // we have two hashmaps: 1. IDAndCent and 2. IDAndID. we will use IDAndCent +//} - - HashMap MapOfIDAndCount = new HashMap(); - - HashMap MapOfIDAndCent = new HashMap(); - - for (Long cur_id : new TreeSet(IDAndCent.keySet())) - { - int cur_count = IDAndCent.get(cur_id).size(); - - MapOfIDAndCount.put(cur_id, (long) cur_count); // this has the hashids and counts. - - List bucketpoints = new ArrayList<>(); - - Iterator e = IDAndCent.get(cur_id).iterator(); - -// int i=1; - while (e.hasNext()) { - -// System.out.println(i++); - - bucketpoints.add(e.next()) ; - - } - - float [] bucketcent; - - bucketcent = medoid(bucketpoints); - - MapOfIDAndCent.put(cur_id, bucketcent); // this has the hashids and centroids. - -// System.out.println(cur_id + " " + cur_count); - - // int c = MapOfIDAndCent.get(cur_id).length; - - // System.out.println(cur_id + " " + c); - - - } - -// int NumberOfMicroClustersBeforePruning = MapOfIDAndCent.size() ; -// System.out.println("NumberOfMicroClustersBeforePruning = "+ NumberOfMicroClustersBeforePruning); + System.out.println("NumberOfMicroClustersBeforePruning = "+ MapOfIDAndCent.size()); // next we want to prune the tree by parent count comparison // follows breadthfirst search - - HashMap denseSetOfIDandCount2 = new HashMap(); for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) @@ -358,25 +299,17 @@ public HashMap findDensityModes2() { } - - //remove keys with support less than 1 - Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); //64 so 6 bits? //stream = stream.filter(p -> p.getKey() > 64); - - List sortedIDList2= new ArrayList<>(); // sort and limit the list - stream2.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*4) + stream2.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*6) .forEachOrdered(x -> sortedIDList2.add(x.getKey())); - - - HashMap KeyAndCent = new HashMap<>(); HashMap KeyAndCount = new HashMap<>(); HashMap WeightAndCent = new HashMap<>(); @@ -389,12 +322,9 @@ public HashMap findDensityModes2() { WeightAndCent.put(MapOfIDAndCount.get(sortedIDList2.get(i)), MapOfIDAndCent.get(sortedIDList2.get(i))); } - - - + return WeightAndCent; - } @@ -413,8 +343,7 @@ public void run() { List weights2 =new ArrayList<>(); - int NumberOfMicroClusters = WeightAndClusters.size() ; - System.out.println("NumberOfMicroClusters = "+ NumberOfMicroClusters); + System.out.println("NumberOfMicroClusters_AfterPruning = "+ WeightAndClusters.size()); // int k = NumberOfMicroClusters>200+so.getk()?200+so.getk():NumberOfMicroClusters; @@ -426,8 +355,7 @@ public void run() { weights2.add((float)weights); centroids2.add(WeightAndClusters.get(weights)); } - - + //System.out.printf("\tvalueofK is "); //System.out.println( so.getk()); @@ -437,20 +365,16 @@ public void run() { aggloOffline.setWeights(weights2); this.centroids = aggloOffline.getCentroids(); - - + } - - - public static void main(String[] args) throws FileNotFoundException, IOException { - int k = 5;//6; - int d = 100;//16; - int n = 5000; + int k = 20;//6; + int d = 500;//16; + int n = 500000; float var = 1.5f; int count = 1; // System.out.printf("ClusterVar\t"); @@ -458,17 +382,21 @@ public static void main(String[] args) throws FileNotFoundException, // System.out.printf("Trial%d\t", i); // System.out.printf("RealWCSS\n"); - String Output = "/C:/Users/user/Desktop/temp/OutputTwrpCents" ; + String Output = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" ; float f = var; float avgrealwcss = 0; float avgtime = 0; // System.out.printf("%f\t", f); GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); - // gen.writeCSVToFile(new - // File("/home/lee/Desktop/reclsh/in.csv")); + + // gen.writeCSVToFile(new File("/home/lee/Desktop/reclsh/in.csv")); + + // List data = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" + RPHashObject o = new SimpleArrayReader(gen.data, k); - o.setDimparameter(8); + + o.setDimparameter(28); TWRPv2 rphit = new TWRPv2(o); long startTime = System.nanoTime(); @@ -481,11 +409,10 @@ public static void main(String[] args) throws FileNotFoundException, VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); - // System.out.printf("%.0f\t", - // StatTests.WCSSECentroidsFloat(centsr, gen.data)); - // System.gc(); + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); - // System.out.printf("%.0f\n", avgrealwcss / count); + System.out.printf("%.0f\n", avgrealwcss / count); } diff --git a/src/main/java/edu/uc/rphash/projections/DBFriendlyProjection.java b/src/main/java/edu/uc/rphash/projections/DBFriendlyProjection.java index 7f676b8..ab1c7b7 100644 --- a/src/main/java/edu/uc/rphash/projections/DBFriendlyProjection.java +++ b/src/main/java/edu/uc/rphash/projections/DBFriendlyProjection.java @@ -108,6 +108,7 @@ public float[] project(float[] v) { // -sqrt(3/t) // n: original dimension // t: target OR projected dimension + static float[] projectN(float[] v, int[][] P, int[][] M, int t) { float[] r = new float[t]; float sum; @@ -117,10 +118,10 @@ static float[] projectN(float[] v, int[][] P, int[][] M, int t) { //------------------------------------------------- // this is what is there in spark code: - for (int col : M[i]) - sum -= v[col]; - for (int col : P[i]) - sum += v[col]; +// for (int col : M[i]) +// sum -= v[col]; +// for (int col : P[i]) +// sum += v[col]; //--------------------------------------------------- @@ -129,13 +130,10 @@ static float[] projectN(float[] v, int[][] P, int[][] M, int t) { for(int j=0;j Date: Thu, 23 Aug 2018 11:01:45 -0400 Subject: [PATCH 3/8] changed MAP to MultiMap, added cmd line parameters. Changed the MAP data structure to MultiMap so that one key can map to multiple values. Added command line parameters to set cutoff and choose zero/random vector. Fixed error for the Rphash Simple, changed the final centroids returned to correct values. --- src/main/java/edu/uc/rphash/RPHash.java | 31 ++++ .../edu/uc/rphash/RPHashAdaptive2Pass.java | 17 +- src/main/java/edu/uc/rphash/RPHashSimple.java | 37 +++- .../edu/uc/rphash/Readers/RPHashObject.java | 9 +- .../uc/rphash/Readers/SimpleArrayReader.java | 32 ++++ .../edu/uc/rphash/Readers/StreamObject.java | 24 +++ src/main/java/edu/uc/rphash/TWRPv2.java | 169 +++++++++++++----- 7 files changed, 266 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/uc/rphash/RPHash.java b/src/main/java/edu/uc/rphash/RPHash.java index 9038026..2098fa5 100644 --- a/src/main/java/edu/uc/rphash/RPHash.java +++ b/src/main/java/edu/uc/rphash/RPHash.java @@ -52,16 +52,24 @@ public class RPHash { static String[] clusteringmethods = { "simple", "streaming", "multiproj", "kmeans", "pkmeans","kmeansplusplus", "streamingkmeans", "adaptive","dummy" ,"twrp"}; + static String[] offlineclusteringmethods = { "singlelink", "completelink", "averagelink", "kmeans", "adaptivemeanshift", "kmpp", "multikmpp" , "dbscan", "none" }; + static String[] projectionmethods = { "dbf", "fjlt", "rp", "svd", "noproj" }; + static String[] ops = { "numprojections", "innerdecodermultiplier", "numblur", "randomseed", "hashmod", "parallel", "streamduration", "raw", "decayrate", "dimparameter", "decodertype", "offlineclusterer", "runs", "normalize", "projection" }; + static String[] decoders = { "dn", "e8", "golay", "multie8", "leech", "multileech", "sphere", "levypstable", "cauchypstable", "gaussianpstable", "adaptive", "origin" }; + + static String[] twrp_options = { "cutoff", "randomvector" }; + + public static void main(String[] args) throws NumberFormatException, IOException, InterruptedException { @@ -95,6 +103,12 @@ public static void main(String[] args) throws NumberFormatException, System.out.print(s + " ,"); System.out.print("]\n"); + System.out.print("\t twrp_options" + "\t:["); + for (String s : twrp_options) + System.out.print(s + " ,"); + System.out.print("]\n"); + + System.exit(0); } @@ -114,6 +128,9 @@ public static void main(String[] args) throws NumberFormatException, matched |= keyword.equals(match); for (String match : decoders) matched |= keyword.equals(match); + for (String match : twrp_options) + matched |= keyword.equals(match); + if (!matched) unmatchedkeywords.add(keyword); } @@ -503,6 +520,20 @@ public static List runConfigs(List untaggedArgs, o.setNormalize(Boolean.parseBoolean(taggedArgs.get("normalize"))); so.setNormalize(Boolean.parseBoolean(taggedArgs.get("normalize"))); } + + + if (taggedArgs.containsKey("cutoff")) { + o.setCutoff(Integer.parseInt(taggedArgs.get("cutoff"))); + so.setCutoff(Integer.parseInt(taggedArgs.get("cutoff"))); + } + + + if (taggedArgs.containsKey("randomvector")) { + o.setRandomVector(Boolean.parseBoolean(taggedArgs.get("randomvector"))); + so.setRandomVector(Boolean.parseBoolean(taggedArgs.get("randomvector"))); + } + + if (taggedArgs.containsKey("projection")) { switch (taggedArgs.get("projection")) { diff --git a/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java b/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java index 88a9637..927bf7f 100644 --- a/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java +++ b/src/main/java/edu/uc/rphash/RPHashAdaptive2Pass.java @@ -223,9 +223,11 @@ public List> findDensityModes() { List sortedIDList= new ArrayList<>(); // sort and limit the list - stream.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*4) + stream.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*6) .forEachOrdered(x -> sortedIDList.add(x.getKey())); + System.out.println("NumberOfMicroClustersAfterPruning = "+ sortedIDList.size()); + // compute centroids HashMap> estcents = new HashMap<>(); @@ -233,6 +235,8 @@ public List> findDensityModes() { { estcents.put(sortedIDList.get(i), IDAndCent.get(sortedIDList.get(i))); } + + // System.out.println(); // for (int i =0; icentroids = new ArrayList<>(); List weights =new ArrayList<>(); - int k = clustermembers.size()>200+so.getk()?200+so.getk():clustermembers.size(); + // int k = clustermembers.size()>200+so.getk()?200+so.getk():clustermembers.size(); + int k = clustermembers.size(); for(int i=0;i centsr = rphit.getCentroids(); diff --git a/src/main/java/edu/uc/rphash/RPHashSimple.java b/src/main/java/edu/uc/rphash/RPHashSimple.java index 2e8cba9..83f7bc2 100644 --- a/src/main/java/edu/uc/rphash/RPHashSimple.java +++ b/src/main/java/edu/uc/rphash/RPHashSimple.java @@ -24,6 +24,7 @@ import edu.uc.rphash.standardhash.NoHash; import edu.uc.rphash.tests.StatTests; import edu.uc.rphash.tests.clusterers.KMeans2; +import edu.uc.rphash.tests.clusterers.MultiKMPP; import edu.uc.rphash.tests.generators.GenerateData; import edu.uc.rphash.tests.generators.GenerateStreamData; import edu.uc.rphash.tests.kmeanspp.KMeansPlusPlus; @@ -54,8 +55,8 @@ public RPHashObject map() { int logk = (int) (.5 + Math.log(so.getk()) / Math.log(2));// log k and // round to // integer - int k = so.getk() * logk; - is = new SimpleFrequentItemSet(k); + int k1 = so.getk() * logk; + is = new SimpleFrequentItemSet(k1); Decoder dec = so.getDecoderType(); dec.setCounter(is); @@ -205,13 +206,27 @@ public void accept(float[] t) { // // } + Clusterer offlineclusterer = so.getOfflineClusterer(); offlineclusterer.setData(centroids); offlineclusterer.setWeights(so.getCounts()); offlineclusterer.setK(so.getk()); + + // System.out.println("\n k sent to offline = "+ so.getk()); + this.centroids = offlineclusterer.getCentroids(); + + //System.out.println("\n cents in reduce from offline cluster = "+ this.centroids.size()); + + //System.out.println("\n cents in reduce after label mapping = "+ centroids.size()); + this.labelmap = VectorUtil.generateIDMap(centroids, this.centroids); - so.setCentroids(centroids); + + //so.setCentroids(centroids); + so.setCentroids(this.centroids); + + + return so; } @@ -272,12 +287,14 @@ private void run() { map(); reduce(); this.centroids = so.getCentroids(); + + } public static void main(String[] args) { int k = 10; - int d = 1000; - int n = 10000; + int d = 200; + int n = 1000; float var = 1f; int count = 5; System.out.printf("Decoder: %s\n", "Sphere"); @@ -296,9 +313,17 @@ public static void main(String[] args) { RPHashSimple rphit = new RPHashSimple(o); o.setDecoderType(new Spherical(32, 4, 1)); // o.setDimparameter(31); - o.setOfflineClusterer(new KMeans2()); + //o.setOfflineClusterer(new KMeans2()); + o.setOfflineClusterer(new MultiKMPP()); + + //System.out.println("\n k sent to offline in MAIN = "+ o.getk()); + long startTime = System.nanoTime(); List centsr = rphit.getCentroids(); + + //System.out.println("\n no of final cents : " + centsr.size()); + + avgtime += (System.nanoTime() - startTime) / 100000000; // avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), diff --git a/src/main/java/edu/uc/rphash/Readers/RPHashObject.java b/src/main/java/edu/uc/rphash/Readers/RPHashObject.java index 5ecb470..2d5b0a5 100644 --- a/src/main/java/edu/uc/rphash/Readers/RPHashObject.java +++ b/src/main/java/edu/uc/rphash/Readers/RPHashObject.java @@ -90,6 +90,13 @@ public interface RPHashObject { void setDimparameter(int parseInt); int getDimparameter(); + void setCutoff(int parseInt); + int getCutoff(); + + void setRandomVector(boolean parseBoolean); + boolean getRandomVector(); + + // void setOfflineClusterer(Clusterer agglomerative3); // Clusterer getOfflineClusterer(); @@ -99,7 +106,7 @@ public interface RPHashObject { int getk(); - void setK(int getk); + void setK(int k); String toString(); void reset();//TODO rename to resetDataStream diff --git a/src/main/java/edu/uc/rphash/Readers/SimpleArrayReader.java b/src/main/java/edu/uc/rphash/Readers/SimpleArrayReader.java index eaf3c2f..8fe1916 100644 --- a/src/main/java/edu/uc/rphash/Readers/SimpleArrayReader.java +++ b/src/main/java/edu/uc/rphash/Readers/SimpleArrayReader.java @@ -33,6 +33,11 @@ public class SimpleArrayReader implements RPHashObject { private Clusterer clusterer; private boolean normalize = false; private Projector projector; + + boolean RandomVector = false; + int Cutoff; + + public void setRandomSeed(long randomSeed) { this.randomSeed = randomSeed; @@ -397,4 +402,31 @@ public void setProjectionType(Projector dbFriendlyProjection) { public Projector getProjectionType(){ return this.projector; } + + + + @Override + public void setCutoff(int parseInt) { + this.Cutoff = parseInt; + + } + + @Override + public int getCutoff() { + + return this.Cutoff; + } + + + + @Override + public void setRandomVector(boolean parseBoolean) { + this.RandomVector = parseBoolean; + } + public boolean getRandomVector() { + return this.RandomVector; + } + + + } diff --git a/src/main/java/edu/uc/rphash/Readers/StreamObject.java b/src/main/java/edu/uc/rphash/Readers/StreamObject.java index 460070f..428a891 100644 --- a/src/main/java/edu/uc/rphash/Readers/StreamObject.java +++ b/src/main/java/edu/uc/rphash/Readers/StreamObject.java @@ -41,6 +41,9 @@ public class StreamObject implements RPHashObject, Iterator { Decoder dec; float decayrate=0; boolean parallel = true; + boolean RandomVector; + int Cutoff; + ExecutorService executor; InputStream inputStream; @@ -425,4 +428,25 @@ public void setProjectionType(Projector dbFriendlyProjection) { public Projector getProjectionType() { return this.projector; } + + + + @Override + public void setCutoff(int parseInt) { + this.Cutoff = parseInt; + } + @Override + public int getCutoff() { + return this.Cutoff; + } + + + @Override + public void setRandomVector(boolean parseBoolean) { + this.RandomVector = parseBoolean; + } + public boolean getRandomVector() { + return this.RandomVector; + } + } diff --git a/src/main/java/edu/uc/rphash/TWRPv2.java b/src/main/java/edu/uc/rphash/TWRPv2.java index d24a616..d98e752 100644 --- a/src/main/java/edu/uc/rphash/TWRPv2.java +++ b/src/main/java/edu/uc/rphash/TWRPv2.java @@ -7,13 +7,13 @@ //import java.util.Arrays; import java.util.HashMap; //import java.util.Iterator; +//import java.util.LinkedHashMap; import java.util.List; +//import java.util.Map; import java.util.Map.Entry; import java.util.Random; import java.util.TreeSet; import java.util.stream.Stream; -//import java.util.Map; - import edu.uc.rphash.Readers.RPHashObject; import edu.uc.rphash.Readers.SimpleArrayReader; @@ -23,6 +23,12 @@ import edu.uc.rphash.tests.generators.GenerateData; import edu.uc.rphash.util.VectorUtil; +//import org.apache.commons.collections.map.MultiValueMap; +//import org.apache.commons.collections.map.*; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + + public class TWRPv2 implements Clusterer, Runnable { @@ -50,7 +56,6 @@ public List getCentroids() { return centroids; } - /* * X - set of vectors compute the medoid of a vector set */ @@ -88,8 +93,7 @@ public static float[][] UpdateHashMap(float cnt_1, float[] x_1, } //float[] rngvec; the range vector is moot if incoming data has been normalized - //post normalization it should all be zero centered, with variance 1 - + //post normalization it should all be zero centered, with variance 1 /* * super simple hash algorithm, reminiscient of pstable lsh */ @@ -127,7 +131,6 @@ public long hashvec2(float[] xt, float[] x, // s <<= 1; s = s << 1 ; // left shift the bits of s by 1. if (xt[i] > rngvec[i]) -// s += 1; s= s+1; if (MapOfIDAndCent.containsKey(s)) { @@ -203,8 +206,8 @@ static boolean isPowerOfTwo(long num) { * density mode via iterative deepening hash counting */ - - public HashMap findDensityModes2() { + public Multimap findDensityModes2() { + //public Map findDensityModes2() { HashMap MapOfIDAndCent = new HashMap<>(); HashMap MapOfIDAndCount = new HashMap<>(); // #create projector matrixs @@ -213,6 +216,7 @@ public HashMap findDensityModes2() { projector.setProjectedDim(so.getDimparameter()); projector.setRandomSeed(so.getRandomSeed()); projector.init(); + int cutoff = so.getCutoff(); int ct = 0; // if(znorm == true){ @@ -299,31 +303,68 @@ public HashMap findDensityModes2() { } - //remove keys with support less than 1 + System.out.println("NumberOfMicroClustersAfterPruning&beforesortingLimit = "+ denseSetOfIDandCount2.size()); + //remove keys with support less than 1 Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); //64 so 6 bits? //stream = stream.filter(p -> p.getKey() > 64); - + +// Stream> stream3 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); +// long counter= stream3.count(); +// System.out.println("NumberOfMicroClustersAfterPruning&limit_the_1s = "+ counter); + +// int cutoff= so.getk()*8; +// if (so.getk()*6 < 210) { cutoff=210+so.getk();} else { cutoff = so.getk()*8;} +// int cutoff = clustermembers.size()>200+so.getk()?200+so.getk():clustermembers.size(); +// System.out.println("Cutoff = "+ cutoff); + List sortedIDList2= new ArrayList<>(); // sort and limit the list - stream2.sorted(Entry. comparingByValue().reversed()).limit(so.getk()*6) + stream2.sorted(Entry. comparingByValue().reversed()).limit(cutoff) .forEachOrdered(x -> sortedIDList2.add(x.getKey())); - - HashMap KeyAndCent = new HashMap<>(); - HashMap KeyAndCount = new HashMap<>(); - HashMap WeightAndCent = new HashMap<>(); - - for (int i =0; i KeyAndCent = new HashMap<>(); +// HashMap KeyAndCount = new HashMap<>(); +// Map WeightAndCent = new HashMap<>(); +// Map WeightAndCent = new LinkedHashMap<>(); + Multimap multimapWeightAndCent = ArrayListMultimap.create(); + + +// for (int i =0; i X) { return ret; } - +// this updates the map two cents with different weigths are merged into one. public static float[][] UpdateHashMap(float cnt_1, float[] x_1, float cnt_2, float[] x_2) { diff --git a/src/main/java/edu/uc/rphash/decoders/SphericalRandom.java b/src/main/java/edu/uc/rphash/decoders/SphericalRandom.java new file mode 100644 index 0000000..32c4250 --- /dev/null +++ b/src/main/java/edu/uc/rphash/decoders/SphericalRandom.java @@ -0,0 +1,292 @@ +package edu.uc.rphash.decoders; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Random; + +import edu.uc.rphash.frequentItemSet.Countable; +import edu.uc.rphash.standardhash.MurmurHash; +import edu.uc.rphash.util.VectorUtil; + +/** + * Spherical LSH Decoder based on SLSH (lgpl) + * + * @author lee + * + */ +public class SphericalRandom implements Decoder { + int HashBits = 32; + final List> vAll; // vAll[i][j] is the vector $A_i \tilde v_j$ + // from + // the article. + int hbits; // Ceil(Log2(2*d)). + int d; // the dimension of the feature space. + int k; // number of elementary hash functions (h) to be concatenated to + // obtain a reliable enough hash function (g). LSH queries becomes + // more selective with increasing k, due to the reduced the + // probability of collision. + int l; // number of "copies" of the bins (with a different random matrices). + // Increasing L will increase the number of points the should be + // scanned linearly during query. + float distance = 0; + + /** + * This class represent a spherical lsh scheme. Vectors are decoded to the + * nearest vertex of the d dimensional orthoplex reresented by a canonical + * ordered integer. + * + * @param d + * - the number of dimension in the orthoplex + * @param k + * - number of rotations of the fundamental hash functions + * @param L + * - the number to search, currently ignored in RPHash + */ + public SphericalRandom(int d, int k, int L) { + this.d = d;// number of dimensions + this.k = k;// number of elementary hash functions + this.l = L;// L;//number of copies to search + double nvertex = 2.0 * this.d; + this.hbits = (int) Math.ceil(Math.log(nvertex) / Math.log(2)); + int kmax = (int) (HashBits / this.hbits); + if (this.k > kmax) { + this.k = kmax; + System.out + .printf("k is too big, chopping down (%d->%d)\n", k, kmax); + } + + Random[] r = new Random[d]; + for (int i = 0; i < d; i++) + r[i] = new Random(); + + // For orthoplex, the basis Vectortors v_i are permutations of the + // Vectortor (1, 0, ..., 0), + // and -(1, 0, ..., 0). + // Thus R v_i simply picks up the ith row of the rotation matrix, up to + // a sign. + // This means we don't need any matrix multiplication; R matrix is the + // list of + // rotated vectors itself! + this.vAll = new ArrayList>(k * l); // random rotation + // matrices + for (int i = 0; i < k * l; i++) { + this.vAll.add(i, randomRotation(this.d, r)); + } + } + + @Override + public int getDimensionality() { + return d; + } + + @Override + public long[] decode(float[] f) { + return Hash(f); + } + + @Override + public float getErrorRadius() { + return d; + } + + @Override + public float getDistance() { + return distance; + } + + long argmaxi(float[] p, int index) { + List vs = vAll.get(index); + long maxi = 0; + float max = 0; + for (int i = 0; i < this.d; i++) { + + float dot = dot(p, vs.get(i)); + // compute orthoplex of -1 and 1 simultaneously + + + //float dot = dotshift(p, vs.get(i)); // aas we are using dotshift the full matrix needs storing. incorporate that. + + float abs = dot >= 0 ? dot : -dot; + if (abs < max) { + continue; + } + max = abs; + maxi = dot >= 0 ? i : i + this.d; + } + return maxi; + } + + float norm(float[] t) { + float n = 0; + for (int i = 0; i < t.length; i++) { + n += t[i] * t[i]; + } + return (float) Math.sqrt(n); + } + + float[] scale(float[] t, float s) { + for (int i = 0; i < t.length; i++) { + t[i] *= s; + } + return t; + } + + float dot(float[] t, float[] u) { + float s = 0; + for (int i = 0; i < t.length; i++) { + s += t[i] * u[i]; + } + return s; + } + + + + float dotshift(float[] t, float[] u) { + float s = 0; + for (int i = 0; i < t.length; i++) { + + + s = (float) ( s + ((t[i]*0.2)+0.1) * u[i] ); + } + return s; + } + + + + float[] sub(float[] t, float[] u) { + for (int i = 0; i < t.length; i++) { + t[i] -= u[i]; + } + return t; + } + + float[] random(int d, Random[] r) { + + float[] v = new float[d]; + + for (int i = 0; i < d; i++) { + v[i] = (float) r[i].nextGaussian(); + } + return v; + } + + List randomRotation(int d, Random[] r2) { + ArrayList R = new ArrayList<>(d); + for (int i = 0; i < d; i++) { + R.add(i, random(d, r2)); + float[] u = R.get(i); + for (int j = 0; j < i; j++) { + float[] v = R.get(j); + float vnorm = norm(v); + if (vnorm == 0) { + return randomRotation(d, r2); + } + float[] vs = new float[v.length]; + System.arraycopy(v, 0, vs, 0, v.length); + scale(vs, dot(v, u) / vnorm); + u = sub(u, vs); + } + u = scale(u, 1.0f / norm(u)); + } + return R; + } + + // Hashes a single point slsh.l times, using a different set of + // random matrices created and stored by the constructor for each. + // Stores the result in g to avoid unnecessary allocations. + // + // SLSH requires that all vectors lie on a d-dimensional hypershpere, + // thus having the same norm. Only the Similarity method of FeatureVector + // is required to take the normalization into account. + // + // The complexity of this function is O(nLK) + long[] Hash(float[] p) { + int ri = 0; + long[] h = new long[l]; + float normp = norm(p); + p = scale(p, 1.0f / normp); + for (int i = 0; i < this.l; i++) { + for (int j = 0; j < this.k; j++) { + h[i] = h[i] | this.argmaxi(p, ri); + h[i] <<= this.hbits; + ri++; + } + } + + return h;//+ (int) (normp); + + } + + public static void main(String[] args) { + Random r = new Random(); + int d = 16; + int K = 3; + int L = 1; + Spherical sp = new Spherical(d, K, L); + + // MultiDecoder sp = new MultiDecoder( d, e8); + MurmurHash hash = new MurmurHash(Integer.MAX_VALUE); + float testResolution = 10000f; + + HashMap ctmap = new HashMap(); + + for (int i = 0; i < 400; i++) { + int ct = 0; + float distavg = 0.0f; + for (int j = 0; j < testResolution; j++) { + float p1[] = new float[d]; + float p2[] = new float[d]; + + // generate a vector + for (int k = 0; k < d; k++) { + p1[k] = r.nextFloat() * 2 - 1f; + p2[k] = (float) (p1[k] + r.nextGaussian() + * ((float) i / 1000f)); + } + float dist = VectorUtil.distance(p1, p2); + distavg += dist; + long[] l1 = sp.decode(p1); + long[] l2 = sp.decode(p2); + + ctmap.put(l1[0], + ctmap.containsKey(l1[0]) ? 1 + ctmap.get(l1[0]) : 1); + + long hp1 = hash.hash(l1); + long hp2 = hash.hash(l2); + + // ctmap.put(hp1,ctmap.containsKey(hp1)?1+ctmap.get(hp1):1); + + ct += (hp2 == hp1) ? 1 : 0; + + } + + System.out.println(distavg / testResolution + "\t" + (float) ct + / testResolution); + } + } + + float[] variance; + +// @Override +// public void setVariance(float[] parameterObject) { +// variance = parameterObject; +// } +// +// @Override +// public float[] getVariance(){ +// return variance; +// } + + @Override + public boolean selfScaling() { + return true; + } + + @Override + public void setCounter(Countable counter) { + // TODO Auto-generated method stub + + } + +} From 3a1cbce174b438f58d0d3fd943cd9c7df80241f5 Mon Sep 17 00:00:00 2001 From: deysn Date: Thu, 15 Aug 2019 14:00:40 -0400 Subject: [PATCH 5/8] TWRPv3 creates 3 trees with diff. random bisection pts and merges them --- .classpath | 2 +- src/main/java/edu/uc/rphash/TWRPv2.java | 2 +- src/main/java/edu/uc/rphash/TWRPv3.java | 607 ++++++++++++++++++++++++ 3 files changed, 609 insertions(+), 2 deletions(-) create mode 100644 src/main/java/edu/uc/rphash/TWRPv3.java diff --git a/.classpath b/.classpath index 8d805ce..0e623b8 100644 --- a/.classpath +++ b/.classpath @@ -4,6 +4,6 @@ - + diff --git a/src/main/java/edu/uc/rphash/TWRPv2.java b/src/main/java/edu/uc/rphash/TWRPv2.java index 3353396..7a025b0 100644 --- a/src/main/java/edu/uc/rphash/TWRPv2.java +++ b/src/main/java/edu/uc/rphash/TWRPv2.java @@ -473,7 +473,7 @@ public static void main(String[] args) throws FileNotFoundException, o.setRandomVector(true); // System.out.println("cutoff = "+ o.getCutoff()); -// System.out.println("get_random_Vector = "+ o.getRandomVector()); + System.out.println("get_random_Vector = "+ o.getRandomVector()); TWRPv2 rphit = new TWRPv2(o); long startTime = System.nanoTime(); diff --git a/src/main/java/edu/uc/rphash/TWRPv3.java b/src/main/java/edu/uc/rphash/TWRPv3.java new file mode 100644 index 0000000..ff221d4 --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRPv3.java @@ -0,0 +1,607 @@ +package edu.uc.rphash; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +//import java.util.Arrays; +import java.util.HashMap; +//import java.util.Iterator; +//import java.util.LinkedHashMap; +import java.util.List; +//import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; + +//import org.apache.commons.collections.map.MultiValueMap; +//import org.apache.commons.collections.map.*; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + + + +public class TWRPv3 implements Clusterer, Runnable { + + boolean znorm = false; + + private int counter; + private float[] rngvec; + private float[] rngvec2; + private float[] rngvec3; + + private List centroids = null; + + private RPHashObject so; + + public TWRPv3(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + return centroids; + } + + + + // combines two hashmaps of idsandcents + + public static HashMap mergehmapsidsandcents( + HashMap partidandcent1, + HashMap partidandcent2, + HashMap partidandcount1, + HashMap partidandcount2) +{ + // new empty map + HashMap combined = new HashMap<>(); + combined.putAll( partidandcent1); + + for(Long key : partidandcent2.keySet()) { + if(combined.containsKey(key)) { + + + Long weight1= partidandcount1.get(key); + + float[] cent1= combined.get(key); + + Long weight2= partidandcount2.get(key); + + float [] cent2= partidandcent2.get(key); + + float [][] joined = UpdateHashMap(weight1, cent1 ,weight2 , cent2 ); + float combinedCount = joined[0][0]; + float [] combinedCent = joined[1]; + + + combined.put(key,combinedCent); + + } + else { + combined.put(key,partidandcent2.get(key)); + } + } + + return (combined); + +} + + + + // combines two hashmaps of idsandcounts + + public static HashMap mergehmapsidsandcounts(HashMap partidandcount1, + HashMap partidandcountt2) + { + + + HashMap combined = new HashMap (); // new empty map + combined.putAll(partidandcount1); + + + + for(Long key : partidandcountt2.keySet()) { + if(combined.containsKey(key)) { + + + long value1 = combined.get(key); + long value2 = partidandcountt2.get(key); + long value3 = value1 + value2; + + combined.put(key,value3); + + + } + else { + combined.put(key,partidandcountt2.get(key)); + } + + + + } + return (combined); + } + + + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + +// this updates the map two cents with different weigths are merged into one. + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, + float cnt_2, float[] x_2) { + + float cnt_r = cnt_1 + cnt_2; + + float[] x_r = new float[x_1.length]; + + for (int i = 0; i < x_1.length; i++) { + x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; + + } + + float[][] ret = new float[2][]; + ret[0] = new float[1]; + ret[0][0] = cnt_r; + ret[1] = x_r; + return ret; + } + + + public long hashvec2( float[] xt, float[] x, + HashMap MapOfIDAndCent, HashMap MapOfIDAndCount,int ct, float[] rngvec) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) + s= s+1; + + if (MapOfIDAndCent.containsKey(s)) { + + float CurrentCount = MapOfIDAndCount.get(s); + float CurrentCent [] = MapOfIDAndCent.get(s); + float CountForIncomingVector = 1; + float IncomingVector [] = x; + + float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, CountForIncomingVector, IncomingVector ); + + Long UpdatedCount = (long) MergedValues[0][0] ; + + float[] MergedVector = MergedValues[1] ; + + MapOfIDAndCount.put(s , UpdatedCount); + + MapOfIDAndCent.put(s, MergedVector); + + } + + else { + + float[] xlist = x; + MapOfIDAndCent.put(s, xlist); + MapOfIDAndCount.put(s, (long)1); + } + } + return s; + } + + + /* + * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid + * vector map + * + * hash the projected vector x and update the hash to centroid and counts + * maps + */ + void addtocounter(float[] x, Projector p, + HashMap IDAndCent,HashMap IDandID,int ct, float[] rngvec) { + float[] xt = p.project(x); + + hashvec2(xt,x,IDAndCent, IDandID,ct,rngvec ); + } + + void addtocounter(float[] x, Projector p, + HashMap IDAndCent,HashMap IDandID,int ct,float[] mean,float[] variance, float[] rngvec ) + { + float[] xt = p.project(StatTests.znormvec(x, mean, variance)); + + + hashvec2(xt,x,IDAndCent, IDandID,ct, rngvec); + } + + static boolean isPowerOfTwo(long num) { + return (num & -num) == num; + } + + + + + /* + * X - data set k - canonical k in k-means l - clustering sub-space Compute + * density mode via iterative deepening hash counting + */ + + + public Multimap findDensityModes2() { + //public Map findDensityModes2() { + HashMap MapOfIDAndCent1 = new HashMap<>(); + HashMap MapOfIDAndCount1 = new HashMap<>(); + + + HashMap MapOfIDAndCent2 = new HashMap<>(); + HashMap MapOfIDAndCount2 = new HashMap<>(); + + + HashMap MapOfIDAndCent3 = new HashMap<>(); + HashMap MapOfIDAndCount3 = new HashMap<>(); + + HashMap MapOfIDAndCent = new HashMap<>(); + HashMap MapOfIDAndCount = new HashMap<>(); + + + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + int cutoff = so.getCutoff(); + + int ct = 0; + + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, MapOfIDAndCent1, MapOfIDAndCount1,ct++, rngvec); + addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec2); + addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec3); + + + } + } + + + MapOfIDAndCount=mergehmapsidsandcounts(MapOfIDAndCount1, MapOfIDAndCount2); + + MapOfIDAndCent = mergehmapsidsandcents(MapOfIDAndCent1,MapOfIDAndCent2,MapOfIDAndCount1, MapOfIDAndCount2 ); + + MapOfIDAndCent = mergehmapsidsandcents(MapOfIDAndCent,MapOfIDAndCent3,MapOfIDAndCount, MapOfIDAndCount3 ); + + MapOfIDAndCount=mergehmapsidsandcounts(MapOfIDAndCount, MapOfIDAndCount3); + + + + + + + System.out.println("NumberOfMicroClustersBeforePruning = "+ MapOfIDAndCent.size()); + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + + + + + HashMap denseSetOfIDandCount2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + +//HashMap> IDAndCent = new HashMap<>(); and HashMap MapOfIDAndCent = new HashMap(); + + MapOfIDAndCent.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + System.out.println("NumberOfMicroClustersAfterPruning&beforesortingLimit = "+ denseSetOfIDandCount2.size()); + + //remove keys with support less than 1 + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + //64 so 6 bits? + //stream = stream.filter(p -> p.getKey() > 64); + +// Stream> stream3 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); +// long counter= stream3.count(); +// System.out.println("NumberOfMicroClustersAfterPruning&limit_the_1s = "+ counter); + +// int cutoff= so.getk()*8; +// if (so.getk()*6 < 210) { cutoff=210+so.getk();} else { cutoff = so.getk()*8;} +// int cutoff = clustermembers.size()>200+so.getk()?200+so.getk():clustermembers.size(); +// System.out.println("Cutoff = "+ cutoff); + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(cutoff) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + +// HashMap KeyAndCent = new HashMap<>(); +// HashMap KeyAndCount = new HashMap<>(); +// Map WeightAndCent = new HashMap<>(); +// Map WeightAndCent = new LinkedHashMap<>(); + Multimap multimapWeightAndCent = ArrayListMultimap.create(); + + + +// + for (Long keys: sortedIDList2) + + { +// WeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + + multimapWeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + +// KeyAndCent.put(keys, MapOfIDAndCent.get(keys)); +// KeyAndCount.put(keys, MapOfIDAndCount.get(keys)); + + } + + + + return multimapWeightAndCent; + +} + + + public void run() { + rngvec = new float[so.getDimparameter()]; + + + rngvec2 = new float[so.getDimparameter()]; + + rngvec3 = new float[so.getDimparameter()]; + + counter = 0; + boolean randVect = so.getRandomVector(); + + // Random r = new Random(so.getRandomSeed()); + // Random r = new Random(3800635955020675334L) ; + Random r = new Random(); + Random r2 = new Random(); + Random r3 = new Random(); + + if (randVect==true){ + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) r.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec2[i] = (float) r2.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec3[i] = (float) r3.nextGaussian(); + + + } else { + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) 0; + } + + + + + + + Multimap WeightAndClusters = findDensityModes2(); + //Map WeightAndClusters = findDensityModes2(); + + + + + Listcentroids2 = new ArrayList<>(); + List weights2 =new ArrayList<>(); + + + System.out.println("NumberOfMicroClusters_AfterPruning = "+ WeightAndClusters.size()); + System.out.println("getRandomVector = "+ randVect); + + // int k = NumberOfMicroClusters>200+so.getk()?200+so.getk():NumberOfMicroClusters; + + // have to prune depending NumberOfMicroClusters returned. + // int i = 1; + // int j=1; + // for (Long weights : new TreeSet(WeightAndClusters.keySet())) + for (Long weights : WeightAndClusters.keys()) + { + // System.out.println("NumberOfTreesetkeys = "+ i); + // String key =weights.toString(); + // System.out.println(weights); + weights2.add((float)weights); + // centroids2.add(WeightAndClusters.get(weights)); + // centroids2.addAll(WeightAndClusters.get(weights)); + // i=i+1; + } + // System.out.println("done printing keys for weights"); + + for (Long weight : WeightAndClusters.keySet()) + + { + // System.out.println(weight); + // System.out.println("NumberOfTreesetkeys = "+ j); + centroids2.addAll(WeightAndClusters.get(weight)); + + // j=j+1; + } + // System.out.println("done printing keys for centroids"); + + // System.out.println(weights2.size()); + // System.out.println(centroids2.size()); + + //System.out.printf("\tvalueofK is "); + //System.out.println( so.getk()); + + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); + + aggloOffline.setWeights(weights2); + + this.centroids = aggloOffline.getCentroids(); + + } + + + public static void main(String[] args) throws FileNotFoundException, + IOException { + + int k = 10;//6; + int d = 700;//16; + int n = 10000; + float var = 1.5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + String Output = "/C:/Users/deysn/Desktop/temp/OutputTwrpCents1" ; + + float f = var; + float avgrealwcss = 0; + float avgtime = 0; + // System.out.printf("%f\t", f); + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); + + // gen.writeCSVToFile(new File("/home/lee/Desktop/reclsh/in.csv")); + + // List data = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" + + RPHashObject o = new SimpleArrayReader(gen.data, k); + + o.setDimparameter(20); + + o.setCutoff(100); + o.setRandomVector(true); + +// System.out.println("cutoff = "+ o.getCutoff()); +// System.out.println("get_random_Vector = "+ o.getRandomVector()); + + TWRPv2 rphit = new TWRPv2(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); + + System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } + + //@Override + public void setCutoff(int getCutoff) { + this.so.setCutoff(getCutoff); + } + + //@Override + public void setRandomVector(boolean getRandomVector) { + this.so.setRandomVector(getRandomVector); + } + + +} From 4b74eec7e2e2e04f2432e3709b88fe0afe307d2b Mon Sep 17 00:00:00 2001 From: deysn Date: Thu, 22 Aug 2019 13:15:32 -0400 Subject: [PATCH 6/8] TWRPv3 : TWRPv3 : --- src/main/java/edu/uc/rphash/TWRPv3.java | 5 +- src/main/java/edu/uc/rphash/TWRPv4.java | 440 +++++++++++++ src/main/java/edu/uc/rphash/TWRPv5_WCSS.java | 621 +++++++++++++++++++ 3 files changed, 1063 insertions(+), 3 deletions(-) create mode 100644 src/main/java/edu/uc/rphash/TWRPv4.java create mode 100644 src/main/java/edu/uc/rphash/TWRPv5_WCSS.java diff --git a/src/main/java/edu/uc/rphash/TWRPv3.java b/src/main/java/edu/uc/rphash/TWRPv3.java index ff221d4..b031fcd 100644 --- a/src/main/java/edu/uc/rphash/TWRPv3.java +++ b/src/main/java/edu/uc/rphash/TWRPv3.java @@ -285,7 +285,7 @@ public Multimap findDensityModes2() { { addtocounter(x, projector, MapOfIDAndCent1, MapOfIDAndCount1,ct++, rngvec); addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec2); - addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec3); + addtocounter(x, projector, MapOfIDAndCent3, MapOfIDAndCount3,ct++, rngvec3); } @@ -405,7 +405,6 @@ public Multimap findDensityModes2() { public void run() { rngvec = new float[so.getDimparameter()]; - rngvec2 = new float[so.getDimparameter()]; rngvec3 = new float[so.getDimparameter()]; @@ -532,7 +531,7 @@ public static void main(String[] args) throws FileNotFoundException, // System.out.println("cutoff = "+ o.getCutoff()); // System.out.println("get_random_Vector = "+ o.getRandomVector()); - TWRPv2 rphit = new TWRPv2(o); + TWRPv3 rphit = new TWRPv3(o); long startTime = System.nanoTime(); List centsr = rphit.getCentroids(); diff --git a/src/main/java/edu/uc/rphash/TWRPv4.java b/src/main/java/edu/uc/rphash/TWRPv4.java new file mode 100644 index 0000000..d0bad1f --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRPv4.java @@ -0,0 +1,440 @@ +package edu.uc.rphash; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + + +public class TWRPv4 implements Clusterer, Runnable { + + boolean znorm = false; + + private int counter; + private List centroids = null; + private float[] bisectionVector; + + private RPHashObject so; + + public TWRPv4(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + return centroids; + } + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + +// this updates the map two cents with different weigths are merged into one. + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, + float cnt_2, float[] x_2) { + + float cnt_r = cnt_1 + cnt_2; + + float[] x_r = new float[x_1.length]; + + for (int i = 0; i < x_1.length; i++) { + x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; + + } + + float[][] ret = new float[2][]; + ret[0] = new float[1]; + ret[0][0] = cnt_r; + ret[1] = x_r; + return ret; + } + + //float[] rngvec; the range vector is moot if incoming data has been normalized + //post normalization it should all be zero centered, with variance 1 + /* + * super simple hash algorithm, reminiscient of pstable lsh + */ + // xt is the projected vector and x is the original vector , rngvec is the randomly generated vector of projected dim. + + + public long hashvec2(float[] xt, float[] x, + HashMap MapOfIDAndCent, HashMap MapOfIDAndCount,int ct, float[] bisectionVector) { + +// for (int i=0 ; i data = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" + + RPHashObject o = new SimpleArrayReader(gen.data, k); + + o.setDimparameter(16); + + o.setCutoff(100); + o.setRandomVector(true); +// System.out.println("cutoff = "+ o.getCutoff()); + System.out.println("get_random_Vector = "+ o.getRandomVector()); + + TWRPv4 rphit = new TWRPv4(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); + + System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } + + //@Override + public void setCutoff(int getCutoff) { + this.so.setCutoff(getCutoff); + } + + //@Override + public void setRandomVector(boolean getRandomVector) { + this.so.setRandomVector(getRandomVector); + } + + +} diff --git a/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java b/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java new file mode 100644 index 0000000..5f5e8d8 --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java @@ -0,0 +1,621 @@ +package edu.uc.rphash; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +//import java.util.Arrays; +import java.util.HashMap; +//import java.util.Iterator; +//import java.util.LinkedHashMap; +import java.util.List; +//import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; + +//import org.apache.commons.collections.map.MultiValueMap; +//import org.apache.commons.collections.map.*; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + + + +public class TWRPv5_WCSS implements Clusterer, Runnable { + + boolean znorm = false; + + private int counter; + private float[] rngvec; + private float[] rngvec2; + private float[] rngvec3; + + private List centroids = null; + + private RPHashObject so; + + public TWRPv5_WCSS(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + return centroids; + } + + + + // combines two hashmaps of idsandcents + + public static HashMap mergehmapsidsandcents( + HashMap partidandcent1, + HashMap partidandcent2, + HashMap partidandcount1, + HashMap partidandcount2) +{ + // new empty map + HashMap combined = new HashMap<>(); + combined.putAll( partidandcent1); + + + + for(Long key : partidandcent2.keySet()) { + if(combined.containsKey(key)) { + + + Long weight1= partidandcount1.get(key); + + float[] cent1= combined.get(key); + + Long weight2= partidandcount2.get(key); + + float [] cent2= partidandcent2.get(key); + + float [][] joined = UpdateHashMap(weight1, cent1 ,weight2 , cent2 ); + float combinedCount = joined[0][0]; + float [] combinedCent = joined[1]; + + + combined.put(key,combinedCent); + + } + else { + combined.put(key,partidandcent2.get(key)); + } + } + + return (combined); + +} + + + + // combines two hashmaps of idsandcounts + + public static HashMap mergehmapsidsandcounts(HashMap partidandcount1, + HashMap partidandcountt2) + { + + + HashMap combined = new HashMap (); // new empty map + combined.putAll(partidandcount1); + + + + for(Long key : partidandcountt2.keySet()) { + if(combined.containsKey(key)) { + + + long value1 = combined.get(key); + long value2 = partidandcountt2.get(key); + long value3 = value1 + value2; + + combined.put(key,value3); + + + } + else { + combined.put(key,partidandcountt2.get(key)); + } + + + + } + return (combined); + } + + + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + +// this updates the map two cents with different weigths are merged into one. + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, + float cnt_2, float[] x_2) { + + float cnt_r = cnt_1 + cnt_2; + + float[] x_r = new float[x_1.length]; + + for (int i = 0; i < x_1.length; i++) { + x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; + + + x_diff_sq[i]= (x_1[i]- x_2[i])*(x_1[i]- x_2[i]); + + + } + + + + float[][] ret = new float[2][]; + ret[0] = new float[1]; + ret[0][0] = cnt_r; + ret[1] = x_r; + return ret; + + + + + } + + + public long hashvec2( float[] xt, float[] x, + HashMap MapOfIDAndCent, HashMap MapOfIDAndCount, int ct, float[] rngvec, HashMap MapOfIDAndWCSS) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) + s= s+1; + + if (MapOfIDAndCent.containsKey(s)) { + + float CurrentCount = MapOfIDAndCount.get(s); + float CurrentCent [] = MapOfIDAndCent.get(s); + float CountForIncomingVector = 1; + float IncomingVector [] = x; + + float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, CountForIncomingVector, IncomingVector ); + + Long UpdatedCount = (long) MergedValues[0][0] ; + + float[] MergedVector = MergedValues[1] ; + + MapOfIDAndCount.put(s , UpdatedCount); + + MapOfIDAndCent.put(s, MergedVector); + + } + + else { + + float[] xlist = x; + MapOfIDAndCent.put(s, xlist); + MapOfIDAndCount.put(s, (long)1); + } + } + return s; + } + + + /* + * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid + * vector map + * + * hash the projected vector x and update the hash to centroid and counts + * maps + */ + void addtocounter(float[] x, Projector p, + HashMap IDAndCent,HashMap IDandID,int ct, float[] rngvec , HashMap IDandWCSS) { + float[] xt = p.project(x); + + hashvec2(xt,x,IDAndCent, IDandID, ct,rngvec , IDandWCSS); + } + + + + static boolean isPowerOfTwo(long num) { + return (num & -num) == num; + } + + + + + /* + * X - data set k - canonical k in k-means l - clustering sub-space Compute + * density mode via iterative deepening hash counting + */ + + + public Multimap findDensityModes2() { + //public Map findDensityModes2() { + HashMap MapOfIDAndCent1 = new HashMap<>(); + HashMap MapOfIDAndCount1 = new HashMap<>(); + + + HashMap MapOfIDAndCent2 = new HashMap<>(); + HashMap MapOfIDAndCount2 = new HashMap<>(); + + + HashMap MapOfIDAndCent3 = new HashMap<>(); + HashMap MapOfIDAndCount3 = new HashMap<>(); + + HashMap MapOfIDAndCent = new HashMap<>(); + HashMap MapOfIDAndCount = new HashMap<>(); + + + HashMap MapOfIDAndWCSS1 = new HashMap<>(); + HashMap MapOfIDAndWCSS2 = new HashMap<>(); + HashMap MapOfIDAndWCSS3 = new HashMap<>(); + HashMap MapOfIDAndWCSS = new HashMap<>(); + + + + + + + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + int cutoff = so.getCutoff(); + + int ct = 0; + + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, MapOfIDAndCent1, MapOfIDAndCount1,ct++, rngvec, MapOfIDAndWCSS1); + addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec2,MapOfIDAndWCSS2); + addtocounter(x, projector, MapOfIDAndCent3, MapOfIDAndCount3,ct++, rngvec3,MapOfIDAndWCSS3); + + + } + } + + + MapOfIDAndCount=mergehmapsidsandcounts(MapOfIDAndCount1, MapOfIDAndCount2); + + MapOfIDAndCent = mergehmapsidsandcents(MapOfIDAndCent1,MapOfIDAndCent2,MapOfIDAndCount1, MapOfIDAndCount2 ); + + MapOfIDAndCent = mergehmapsidsandcents(MapOfIDAndCent,MapOfIDAndCent3,MapOfIDAndCount, MapOfIDAndCount3 ); + + MapOfIDAndCount=mergehmapsidsandcounts(MapOfIDAndCount, MapOfIDAndCount3); + + + + + + + System.out.println("NumberOfMicroClustersBeforePruning = "+ MapOfIDAndCent.size()); + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + + + + + HashMap denseSetOfIDandCount2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + +//HashMap> IDAndCent = new HashMap<>(); and HashMap MapOfIDAndCent = new HashMap(); + + MapOfIDAndCent.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + System.out.println("NumberOfMicroClustersAfterPruning&beforesortingLimit = "+ denseSetOfIDandCount2.size()); + + //remove keys with support less than 1 + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + //64 so 6 bits? + //stream = stream.filter(p -> p.getKey() > 64); + +// Stream> stream3 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); +// long counter= stream3.count(); +// System.out.println("NumberOfMicroClustersAfterPruning&limit_the_1s = "+ counter); + +// int cutoff= so.getk()*8; +// if (so.getk()*6 < 210) { cutoff=210+so.getk();} else { cutoff = so.getk()*8;} +// int cutoff = clustermembers.size()>200+so.getk()?200+so.getk():clustermembers.size(); +// System.out.println("Cutoff = "+ cutoff); + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(cutoff) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + +// HashMap KeyAndCent = new HashMap<>(); +// HashMap KeyAndCount = new HashMap<>(); +// Map WeightAndCent = new HashMap<>(); +// Map WeightAndCent = new LinkedHashMap<>(); + Multimap multimapWeightAndCent = ArrayListMultimap.create(); + + + +// + for (Long keys: sortedIDList2) + + { +// WeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + + multimapWeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + +// KeyAndCent.put(keys, MapOfIDAndCent.get(keys)); +// KeyAndCount.put(keys, MapOfIDAndCount.get(keys)); + + } + + + + return multimapWeightAndCent; + +} + + + public void run() { + rngvec = new float[so.getDimparameter()]; + + rngvec2 = new float[so.getDimparameter()]; + + rngvec3 = new float[so.getDimparameter()]; + + counter = 0; + boolean randVect = so.getRandomVector(); + + // Random r = new Random(so.getRandomSeed()); + // Random r = new Random(3800635955020675334L) ; + Random r = new Random(); + Random r2 = new Random(); + Random r3 = new Random(); + + if (randVect==true){ + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) r.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec2[i] = (float) r2.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec3[i] = (float) r3.nextGaussian(); + + + } else { + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) 0; + } + + + + + + + Multimap WeightAndClusters = findDensityModes2(); + //Map WeightAndClusters = findDensityModes2(); + + + + + Listcentroids2 = new ArrayList<>(); + List weights2 =new ArrayList<>(); + + + System.out.println("NumberOfMicroClusters_AfterPruning = "+ WeightAndClusters.size()); + System.out.println("getRandomVector = "+ randVect); + + // int k = NumberOfMicroClusters>200+so.getk()?200+so.getk():NumberOfMicroClusters; + + // have to prune depending NumberOfMicroClusters returned. + // int i = 1; + // int j=1; + // for (Long weights : new TreeSet(WeightAndClusters.keySet())) + for (Long weights : WeightAndClusters.keys()) + { + // System.out.println("NumberOfTreesetkeys = "+ i); + // String key =weights.toString(); + // System.out.println(weights); + weights2.add((float)weights); + // centroids2.add(WeightAndClusters.get(weights)); + // centroids2.addAll(WeightAndClusters.get(weights)); + // i=i+1; + } + // System.out.println("done printing keys for weights"); + + for (Long weight : WeightAndClusters.keySet()) + + { + // System.out.println(weight); + // System.out.println("NumberOfTreesetkeys = "+ j); + centroids2.addAll(WeightAndClusters.get(weight)); + + // j=j+1; + } + // System.out.println("done printing keys for centroids"); + + // System.out.println(weights2.size()); + // System.out.println(centroids2.size()); + + //System.out.printf("\tvalueofK is "); + //System.out.println( so.getk()); + + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); + + aggloOffline.setWeights(weights2); + + this.centroids = aggloOffline.getCentroids(); + + } + + + public static void main(String[] args) throws FileNotFoundException, + IOException { + + int k = 10;//6; + int d = 700;//16; + int n = 10000; + float var = 1.5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + String Output = "/C:/Users/deysn/Desktop/temp/OutputTwrpCents1" ; + + float f = var; + float avgrealwcss = 0; + float avgtime = 0; + // System.out.printf("%f\t", f); + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); + + // gen.writeCSVToFile(new File("/home/lee/Desktop/reclsh/in.csv")); + + // List data = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" + + RPHashObject o = new SimpleArrayReader(gen.data, k); + + o.setDimparameter(20); + + o.setCutoff(100); + o.setRandomVector(true); + +// System.out.println("cutoff = "+ o.getCutoff()); +// System.out.println("get_random_Vector = "+ o.getRandomVector()); + + TWRPv5_WCSS rphit = new TWRPv5_WCSS(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); + + System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } + + //@Override + public void setCutoff(int getCutoff) { + this.so.setCutoff(getCutoff); + } + + //@Override + public void setRandomVector(boolean getRandomVector) { + this.so.setRandomVector(getRandomVector); + } + + +} From 8646ed50adb2d034f5b5baece557e0c7633fa5ff Mon Sep 17 00:00:00 2001 From: deysn Date: Thu, 5 Sep 2019 08:38:42 -0400 Subject: [PATCH 7/8] added the selection of the best tree having best wcss. can select trwp version fron cmd line. --- src/main/java/edu/uc/rphash/RPHash.java | 19 +- src/main/java/edu/uc/rphash/TWRPv5_WCSS.java | 263 +++++++------------ 2 files changed, 112 insertions(+), 170 deletions(-) diff --git a/src/main/java/edu/uc/rphash/RPHash.java b/src/main/java/edu/uc/rphash/RPHash.java index 2098fa5..618031c 100644 --- a/src/main/java/edu/uc/rphash/RPHash.java +++ b/src/main/java/edu/uc/rphash/RPHash.java @@ -51,7 +51,7 @@ public class RPHash { static String[] clusteringmethods = { "simple", "streaming", "multiproj", - "kmeans", "pkmeans","kmeansplusplus", "streamingkmeans", "adaptive","dummy" ,"twrp"}; + "kmeans", "pkmeans","kmeansplusplus", "streamingkmeans", "adaptive","dummy" ,"twrp" , "twrpbisect", "twrpbest", "twrpmergetree" }; static String[] offlineclusteringmethods = { "singlelink", "completelink", "averagelink", "kmeans", "adaptivemeanshift", "kmpp", "multikmpp" , "dbscan", "none" }; @@ -757,6 +757,23 @@ public static List runConfigs(List untaggedArgs, break; } + case "twrpmergetree": { + runitems.add(new TWRPv3(o)); + break; + } + + case "twrpbisect": { + runitems.add(new TWRPv4(o)); + break; + } + + case "twrpbest": { + runitems.add(new TWRPv5_WCSS(o)); + break; + } + + + case "dummy": { runitems.add(new DummyClusterer(so)); break; diff --git a/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java b/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java index 5f5e8d8..62e9e1f 100644 --- a/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java +++ b/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java @@ -61,86 +61,7 @@ public List getCentroids() { - // combines two hashmaps of idsandcents - - public static HashMap mergehmapsidsandcents( - HashMap partidandcent1, - HashMap partidandcent2, - HashMap partidandcount1, - HashMap partidandcount2) -{ - // new empty map - HashMap combined = new HashMap<>(); - combined.putAll( partidandcent1); - - - - for(Long key : partidandcent2.keySet()) { - if(combined.containsKey(key)) { - - - Long weight1= partidandcount1.get(key); - - float[] cent1= combined.get(key); - - Long weight2= partidandcount2.get(key); - - float [] cent2= partidandcent2.get(key); - - float [][] joined = UpdateHashMap(weight1, cent1 ,weight2 , cent2 ); - float combinedCount = joined[0][0]; - float [] combinedCent = joined[1]; - - - combined.put(key,combinedCent); - - } - else { - combined.put(key,partidandcent2.get(key)); - } - } - - return (combined); - -} - - - - // combines two hashmaps of idsandcounts - - public static HashMap mergehmapsidsandcounts(HashMap partidandcount1, - HashMap partidandcountt2) - { - - - HashMap combined = new HashMap (); // new empty map - combined.putAll(partidandcount1); - - - - for(Long key : partidandcountt2.keySet()) { - if(combined.containsKey(key)) { - - - long value1 = combined.get(key); - long value2 = partidandcountt2.get(key); - long value3 = value1 + value2; - - combined.put(key,value3); - - - } - else { - combined.put(key,partidandcountt2.get(key)); - } - - - - } - return (combined); - } - - + /* /* * X - set of vectors compute the medoid of a vector set @@ -159,32 +80,49 @@ float[] medoid(List X) { } // this updates the map two cents with different weigths are merged into one. - public static float[][] UpdateHashMap(float cnt_1, float[] x_1, - float cnt_2, float[] x_2) { + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, float wcss_1, + float cnt_2, float[] x_2 , float wcss_2) { float cnt_r = cnt_1 + cnt_2; float[] x_r = new float[x_1.length]; + float[] var_r1 = new float[x_1.length]; + float[] var_r2 = new float[x_1.length]; + + double var1=0; + double var2=0; + + for (int i = 0; i < x_1.length; i++) { x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; - - x_diff_sq[i]= (x_1[i]- x_2[i])*(x_1[i]- x_2[i]); - - + var_r1[i] = ((-x_r[i] + x_1[i]) * (-x_r[i] + x_1[i]))/1000000000; + + var_r2[i] =(((-x_r[i] + x_2[i]) * (-x_r[i] + x_2[i])))/1000000000; + } - - - float[][] ret = new float[2][]; + for (int i = 0; i < var_r1.length; i++) { + var1 = var1 + var_r1[i]; + + var2 = var2 + var_r2[i]; + } + double wcsse=0; + wcsse = ( cnt_1*(wcss_1*wcss_1 + (var1)) + var2 / (cnt_1 + cnt_2 ) ) ; + + // System.out.println("wcsse = " + wcsse); + + float wcss = (float) wcsse; + + float[][] ret = new float[3][]; ret[0] = new float[1]; ret[0][0] = cnt_r; ret[1] = x_r; + ret[2]= new float [1]; + ret[2][0]= wcss; return ret; - - - + } @@ -204,16 +142,24 @@ public long hashvec2( float[] xt, float[] x, float CurrentCent [] = MapOfIDAndCent.get(s); float CountForIncomingVector = 1; float IncomingVector [] = x; + float currentWcss= MapOfIDAndWCSS.get(s); + float incomingWcss= 0; + - float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, CountForIncomingVector, IncomingVector ); + float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, currentWcss, CountForIncomingVector, IncomingVector, incomingWcss ); Long UpdatedCount = (long) MergedValues[0][0] ; float[] MergedVector = MergedValues[1] ; + float wcss= MergedValues[2][0]; + MapOfIDAndCount.put(s , UpdatedCount); MapOfIDAndCent.put(s, MergedVector); + + MapOfIDAndWCSS.put(s, wcss); + } @@ -222,6 +168,7 @@ public long hashvec2( float[] xt, float[] x, float[] xlist = x; MapOfIDAndCent.put(s, xlist); MapOfIDAndCount.put(s, (long)1); + MapOfIDAndWCSS.put(s, (float)0); } } return s; @@ -243,12 +190,10 @@ void addtocounter(float[] x, Projector p, } - static boolean isPowerOfTwo(long num) { return (num & -num) == num; } - /* @@ -261,27 +206,20 @@ public Multimap findDensityModes2() { //public Map findDensityModes2() { HashMap MapOfIDAndCent1 = new HashMap<>(); HashMap MapOfIDAndCount1 = new HashMap<>(); - + HashMap MapOfIDAndWCSS1 = new HashMap<>(); HashMap MapOfIDAndCent2 = new HashMap<>(); HashMap MapOfIDAndCount2 = new HashMap<>(); - + HashMap MapOfIDAndWCSS2 = new HashMap<>(); HashMap MapOfIDAndCent3 = new HashMap<>(); HashMap MapOfIDAndCount3 = new HashMap<>(); + HashMap MapOfIDAndWCSS3 = new HashMap<>(); HashMap MapOfIDAndCent = new HashMap<>(); HashMap MapOfIDAndCount = new HashMap<>(); - - - HashMap MapOfIDAndWCSS1 = new HashMap<>(); - HashMap MapOfIDAndWCSS2 = new HashMap<>(); - HashMap MapOfIDAndWCSS3 = new HashMap<>(); HashMap MapOfIDAndWCSS = new HashMap<>(); - - - - + // #create projector matrixs @@ -305,19 +243,49 @@ public Multimap findDensityModes2() { } } + + float WCSS1 = 0; + float WCSS2 = 0; + float WCSS3 = 0; - MapOfIDAndCount=mergehmapsidsandcounts(MapOfIDAndCount1, MapOfIDAndCount2); - - MapOfIDAndCent = mergehmapsidsandcents(MapOfIDAndCent1,MapOfIDAndCent2,MapOfIDAndCount1, MapOfIDAndCount2 ); - - MapOfIDAndCent = mergehmapsidsandcents(MapOfIDAndCent,MapOfIDAndCent3,MapOfIDAndCount, MapOfIDAndCount3 ); - MapOfIDAndCount=mergehmapsidsandcounts(MapOfIDAndCount, MapOfIDAndCount3); + for (Long cur_id : (MapOfIDAndWCSS1.keySet())) + + { // System.out.println("wcss1 = " + MapOfIDAndWCSS1.get(cur_id)); + WCSS1 = WCSS1 + MapOfIDAndWCSS1.get(cur_id);} + for (Long cur_id : (MapOfIDAndWCSS2.keySet())) + + { WCSS2 = WCSS2 + MapOfIDAndWCSS2.get(cur_id);} + for (Long cur_id : (MapOfIDAndWCSS3.keySet())) + + { WCSS3 = WCSS3 + MapOfIDAndWCSS3.get(cur_id);} +// System.out.println("wcss1 = " + WCSS1); +// System.out.println("wcss2 = " + WCSS2); +// System.out.println("wcss3 = " + WCSS3); + if ((WCSS1 >= WCSS2) && (WCSS1>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount1; + MapOfIDAndCent = MapOfIDAndCent1; + MapOfIDAndWCSS = MapOfIDAndWCSS1; + System.out.println("winner = tree1"); + } + else if ((WCSS2 >= WCSS1) && (WCSS2>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount2; + MapOfIDAndCent = MapOfIDAndCent2; + MapOfIDAndWCSS = MapOfIDAndWCSS2; + System.out.println("winner = tree2"); + } + else + {MapOfIDAndCount = MapOfIDAndCount3; + MapOfIDAndCent = MapOfIDAndCent3; + MapOfIDAndWCSS = MapOfIDAndWCSS3; + System.out.println("winner = tree3"); + + } System.out.println("NumberOfMicroClustersBeforePruning = "+ MapOfIDAndCent.size()); @@ -327,7 +295,6 @@ public Multimap findDensityModes2() { - HashMap denseSetOfIDandCount2 = new HashMap(); for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) { @@ -342,8 +309,6 @@ public Multimap findDensityModes2() { denseSetOfIDandCount2.put(parent_id, 0L); // IDAndCent.put(parent_id, new ArrayList<>()); -//HashMap> IDAndCent = new HashMap<>(); and HashMap MapOfIDAndCent = new HashMap(); - MapOfIDAndCent.put(parent_id, new float[]{}); // MapOfIDAndCount.put(parent_id, new Long (0)); @@ -372,45 +337,27 @@ public Multimap findDensityModes2() { //remove keys with support less than 1 Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); - //64 so 6 bits? - //stream = stream.filter(p -> p.getKey() > 64); - -// Stream> stream3 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); -// long counter= stream3.count(); -// System.out.println("NumberOfMicroClustersAfterPruning&limit_the_1s = "+ counter); - -// int cutoff= so.getk()*8; -// if (so.getk()*6 < 210) { cutoff=210+so.getk();} else { cutoff = so.getk()*8;} -// int cutoff = clustermembers.size()>200+so.getk()?200+so.getk():clustermembers.size(); -// System.out.println("Cutoff = "+ cutoff); + List sortedIDList2= new ArrayList<>(); // sort and limit the list stream2.sorted(Entry. comparingByValue().reversed()).limit(cutoff) .forEachOrdered(x -> sortedIDList2.add(x.getKey())); -// HashMap KeyAndCent = new HashMap<>(); -// HashMap KeyAndCount = new HashMap<>(); -// Map WeightAndCent = new HashMap<>(); -// Map WeightAndCent = new LinkedHashMap<>(); - Multimap multimapWeightAndCent = ArrayListMultimap.create(); - + Multimap multimapWeightAndCent = ArrayListMultimap.create(); -// + for (Long keys: sortedIDList2) { -// WeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + multimapWeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); - -// KeyAndCent.put(keys, MapOfIDAndCent.get(keys)); -// KeyAndCount.put(keys, MapOfIDAndCount.get(keys)); + } - return multimapWeightAndCent; @@ -451,13 +398,8 @@ public void run() { - - Multimap WeightAndClusters = findDensityModes2(); - //Map WeightAndClusters = findDensityModes2(); - - Listcentroids2 = new ArrayList<>(); @@ -467,40 +409,23 @@ public void run() { System.out.println("NumberOfMicroClusters_AfterPruning = "+ WeightAndClusters.size()); System.out.println("getRandomVector = "+ randVect); - // int k = NumberOfMicroClusters>200+so.getk()?200+so.getk():NumberOfMicroClusters; - - // have to prune depending NumberOfMicroClusters returned. - // int i = 1; - // int j=1; - // for (Long weights : new TreeSet(WeightAndClusters.keySet())) + for (Long weights : WeightAndClusters.keys()) { - // System.out.println("NumberOfTreesetkeys = "+ i); - // String key =weights.toString(); - // System.out.println(weights); + weights2.add((float)weights); - // centroids2.add(WeightAndClusters.get(weights)); - // centroids2.addAll(WeightAndClusters.get(weights)); - // i=i+1; + } - // System.out.println("done printing keys for weights"); + for (Long weight : WeightAndClusters.keySet()) { - // System.out.println(weight); - // System.out.println("NumberOfTreesetkeys = "+ j); + centroids2.addAll(WeightAndClusters.get(weight)); - // j=j+1; } - // System.out.println("done printing keys for centroids"); - - // System.out.println(weights2.size()); - // System.out.println(centroids2.size()); - - //System.out.printf("\tvalueofK is "); - //System.out.println( so.getk()); + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); @@ -515,8 +440,8 @@ public static void main(String[] args) throws FileNotFoundException, IOException { int k = 10;//6; - int d = 700;//16; - int n = 10000; + int d = 200;//16; + int n = 100000; float var = 1.5f; int count = 1; // System.out.printf("ClusterVar\t"); @@ -538,7 +463,7 @@ public static void main(String[] args) throws FileNotFoundException, RPHashObject o = new SimpleArrayReader(gen.data, k); - o.setDimparameter(20); + o.setDimparameter(16); o.setCutoff(100); o.setRandomVector(true); From c7f92944c137eb0d7073ff4f63b9058dcf114c58 Mon Sep 17 00:00:00 2001 From: deysn Date: Thu, 17 Oct 2019 16:13:04 -0400 Subject: [PATCH 8/8] algorithms to choose best tree --- src/main/java/edu/uc/rphash/RPHash.java | 21 +- .../uc/rphash/RPHashSimple_multiPosLsh.java | 384 +++++++++++ src/main/java/edu/uc/rphash/TWRPv5_WCSS.java | 3 +- src/main/java/edu/uc/rphash/TWRPv6_COV.java | 549 +++++++++++++++ src/main/java/edu/uc/rphash/TWRPv6_WCSS2.java | 626 ++++++++++++++++++ .../edu/uc/rphash/TWRPv6_meanVariance.java | 549 +++++++++++++++ 6 files changed, 2128 insertions(+), 4 deletions(-) create mode 100644 src/main/java/edu/uc/rphash/RPHashSimple_multiPosLsh.java create mode 100644 src/main/java/edu/uc/rphash/TWRPv6_COV.java create mode 100644 src/main/java/edu/uc/rphash/TWRPv6_WCSS2.java create mode 100644 src/main/java/edu/uc/rphash/TWRPv6_meanVariance.java diff --git a/src/main/java/edu/uc/rphash/RPHash.java b/src/main/java/edu/uc/rphash/RPHash.java index 618031c..b0ed7be 100644 --- a/src/main/java/edu/uc/rphash/RPHash.java +++ b/src/main/java/edu/uc/rphash/RPHash.java @@ -51,7 +51,8 @@ public class RPHash { static String[] clusteringmethods = { "simple", "streaming", "multiproj", - "kmeans", "pkmeans","kmeansplusplus", "streamingkmeans", "adaptive","dummy" ,"twrp" , "twrpbisect", "twrpbest", "twrpmergetree" }; + "kmeans", "pkmeans","kmeansplusplus", "streamingkmeans", "adaptive","dummy" ,"twrp" , "twrpbisect", "twrpbest", "twrpmergetree", "twrpbest_afterpruning", + "twrpbest_cov","twrpbest_meanvariance" }; static String[] offlineclusteringmethods = { "singlelink", "completelink", "averagelink", "kmeans", "adaptivemeanshift", "kmpp", "multikmpp" , "dbscan", "none" }; @@ -772,7 +773,23 @@ public static List runConfigs(List untaggedArgs, break; } - + case "twrpbest_afterpruning": { + runitems.add(new TWRPv6_WCSS2(o)); + break; + } + + case "twrpbest_cov": { + runitems.add(new TWRPv6_COV(o)); + break; + } + + case "twrpbest_meanvariance": { + runitems.add(new TWRPv6_meanVariance(o)); + break; + } + + + case "dummy": { runitems.add(new DummyClusterer(so)); diff --git a/src/main/java/edu/uc/rphash/RPHashSimple_multiPosLsh.java b/src/main/java/edu/uc/rphash/RPHashSimple_multiPosLsh.java new file mode 100644 index 0000000..7a37edf --- /dev/null +++ b/src/main/java/edu/uc/rphash/RPHashSimple_multiPosLsh.java @@ -0,0 +1,384 @@ +package edu.uc.rphash; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.function.Consumer; + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.decoders.Decoder; +import edu.uc.rphash.decoders.DepthProbingLSH; +import edu.uc.rphash.decoders.Leech; +import edu.uc.rphash.decoders.Spherical; +import edu.uc.rphash.decoders.SphericalRandom; +import edu.uc.rphash.frequentItemSet.ItemSet; +import edu.uc.rphash.frequentItemSet.SimpleFrequentItemSet; +import edu.uc.rphash.lsh.LSH; +//import edu.uc.rphash.projections.DBFriendlyProjection; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.standardhash.HashAlgorithm; +import edu.uc.rphash.standardhash.NoHash; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.KMeans2; +import edu.uc.rphash.tests.clusterers.MultiKMPP; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.tests.generators.GenerateStreamData; +import edu.uc.rphash.tests.kmeanspp.KMeansPlusPlus; +import edu.uc.rphash.util.VectorUtil; + +public class RPHashSimple_multiPosLsh implements Clusterer { + // float variance; + + public ItemSet is; + + List labels; + HashMap labelmap; + + public static void mapfunc(float[] vec, LSH lshfunc, ItemSet is) { + + long hash = lshfunc.lshHash(vec); + is.add(hash); + } + + public RPHashObject map() { + + // create our LSH Machine + HashAlgorithm hal = new NoHash(so.getHashmod()); + Iterator vecs = so.getVectorIterator(); + if (!vecs.hasNext()) + return so; + + int logk = (int) (.5 + Math.log(so.getk()) / Math.log(2));// log k and + // round to + // integer + int k1 = so.getk() * logk; + is = new SimpleFrequentItemSet(k1); + Decoder dec = so.getDecoderType(); + dec.setCounter(is); + + Projector p = so.getProjectionType(); + p.setOrigDim(so.getdim()); + p.setProjectedDim(dec.getDimensionality()); + p.setRandomSeed(so.getRandomSeed()); + p.init(); + // no noise to start with + List noise = LSH.genNoiseTable( + dec.getDimensionality(), + so.getNumBlur(), + new Random(), + dec.getErrorRadius() + / (dec.getDimensionality() * dec.getDimensionality())); + + LSH lshfunc = new LSH(dec, p, hal, noise, so.getNormalize()); + + // add to frequent itemset the hashed Decoded randomly projected vector + + if (so.getParallel()) { + List data = so.getRawData(); + data.parallelStream().forEach(new Consumer() { + + @Override + public void accept(float[] t) { + mapfunc(t, lshfunc, is); + } + }); + } + while (vecs.hasNext()) { + mapfunc(vecs.next(), lshfunc, is); + } + // } + // while (vecs.hasNext()) { + // float[] vec = vecs.next(); + // long hash = lshfunc.lshHash(vec); + // is.add(hash); + // + // } + + List topids = is.getTop(); + so.setPreviousTopID(topids); + + List topsizes = is.getCounts(); + + List countsAsFloats = new ArrayList(); + for (long ct : topsizes) + countsAsFloats.add((float) ct); + so.setCounts(countsAsFloats); + return so; + } + + public static void redFunc(float[] vec, LSH lshfunc, List noise, + List labels, List centroids) { + long[] hash = lshfunc.lshHashRadius(vec, noise); + labels.add(-1l); + // radius probe around the vector + for (Centroid cent : centroids) { + for (long h : hash) { + if (cent.ids.contains(h)) { + cent.updateVec(vec); + labels.set(labels.size() - 1, cent.id); + } + } + } + } + + /* + * This is the second phase after the top ids have been in the reduce phase + * aggregated + */ + public RPHashObject reduce() { + + Iterator vecs = so.getVectorIterator(); + if (!vecs.hasNext()) + return so; + float[] vec = vecs.next(); + + HashAlgorithm hal = new NoHash(so.getHashmod()); + Decoder dec = so.getDecoderType(); + + Projector p = so.getProjectionType(); + p.setOrigDim(so.getdim()); + p.setProjectedDim(dec.getDimensionality()); + p.setRandomSeed(so.getRandomSeed()); + p.init(); + + List noise = LSH.genNoiseTable( + so.getdim(), + so.getNumBlur(), + new Random(so.getRandomSeed()), + (float) (dec.getErrorRadius()) + / (float) (dec.getDimensionality() * dec + .getDimensionality())); + + LSH lshfunc = new LSH(dec, p, hal, noise, so.getNormalize()); + List centroids = new ArrayList(); + + for (long id : so.getPreviousTopID()) { + centroids.add(new Centroid(so.getdim(), id, -1)); + } + + this.labels = new ArrayList<>(); + + if (so.getParallel()) { + try { + List data = so.getRawData(); + ForkJoinPool myPool = new ForkJoinPool(this.threads); + myPool.submit(() -> + + data.parallelStream().forEach(new Consumer() { + + @Override + public void accept(float[] t) { + redFunc(t, lshfunc, noise, labels, centroids); + } + + })).get(); + + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + + } else { + while (vecs.hasNext()) { + redFunc(vecs.next(), lshfunc, noise, labels, centroids); + } + } + + // while (vecs.hasNext()) + // { + // + // long[] hash = lshfunc.lshHashRadius(vec, noise); + // labels.add(-1l); + // //radius probe around the vector + // for (Centroid cent : centroids) { + // for (long h : hash) + // { + // if (cent.ids.contains(h)) { + // cent.updateVec(vec); + // this.labels.set(labels.size()-1,cent.id); + // } + // } + // } + // vec = vecs.next(); + // + // } + + + Clusterer offlineclusterer = so.getOfflineClusterer(); + offlineclusterer.setData(centroids); + offlineclusterer.setWeights(so.getCounts()); + offlineclusterer.setK(so.getk()); + + // System.out.println("\n k sent to offline = "+ so.getk()); + + this.centroids = offlineclusterer.getCentroids(); + + //System.out.println("\n cents in reduce from offline cluster = "+ this.centroids.size()); + + //System.out.println("\n cents in reduce after label mapping = "+ centroids.size()); + + this.labelmap = VectorUtil.generateIDMap(centroids, this.centroids); + + //so.setCentroids(centroids); + so.setCentroids(this.centroids); + + + + return so; + } + + // 271458 + // 264779.7 + + public List getLabels() { + for (int i = 0; i < labels.size(); i++) { + if (labelmap.containsKey(labels.get(i))) { + labels.set(i, labelmap.get(labels.get(i))); + } else { + labels.set(i, -1l); + } + } + return this.labels; + } + + private List centroids = null; + private RPHashObject so; + + public RPHashSimple_multiPosLsh(List data, int k) { + so = new SimpleArrayReader(data, k); + } + + int threads = 1; + + public RPHashSimple_multiPosLsh(List data, int k, int processors) { + // System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism",String.valueOf(processors)); + threads = processors; + so = new SimpleArrayReader(data, k); + so.setParallel(true); + } + + public RPHashSimple_multiPosLsh(List data, int k, int times, int rseed) { + so = new SimpleArrayReader(data, k); + } + + public RPHashSimple_multiPosLsh(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + if (centroids == null) + run(); + return centroids; + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + + return centroids; + } + + private void run() { + map(); + reduce(); + this.centroids = so.getCentroids(); + + + } + + public static void main(String[] args) { + int k = 10; + int d = 200; + int n = 1000; + float var = 1f; + int count = 5; + System.out.printf("Decoder: %s\n", "Sphere"); + System.out.printf("ClusterVar\t"); + for (int i = 0; i < count; i++) + System.out.printf("Trial%d\t", i); + System.out.printf("RealWCSS\n"); + + for (float f = var; f < 3.01; f += .05f) { + float avgrealwcss = 0; + float avgtime = 0; + System.out.printf("%f\t", f); + for (int i = 0; i < count; i++) { + GenerateData gen = new GenerateData(k, n / k, d, f, true, 1f); + RPHashObject o = new SimpleArrayReader(gen.data, k); + RPHashSimple_multiPosLsh rphit = new RPHashSimple_multiPosLsh(o); + o.setDecoderType(new SphericalRandom(32, 4, 1)); + //o.setDecoderType(new Spherical(32, 4, 1)); + // o.setDimparameter(31); + //o.setOfflineClusterer(new KMeans2()); + o.setOfflineClusterer(new MultiKMPP()); + + //System.out.println("\n k sent to offline in MAIN = "+ o.getk()); + + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + //System.out.println("\n no of final cents : " + centsr.size()); + + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(),gen.getData()); + + System.out.printf("%.0f\t", + StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); + + } + System.out.printf("%.0f\n", avgrealwcss / count); + } + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + // TODO Auto-generated method stub + + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return true; + } +} diff --git a/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java b/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java index 62e9e1f..4559d38 100644 --- a/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java +++ b/src/main/java/edu/uc/rphash/TWRPv5_WCSS.java @@ -59,8 +59,7 @@ public List getCentroids() { return centroids; } - - + /* /* diff --git a/src/main/java/edu/uc/rphash/TWRPv6_COV.java b/src/main/java/edu/uc/rphash/TWRPv6_COV.java new file mode 100644 index 0000000..f30e517 --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRPv6_COV.java @@ -0,0 +1,549 @@ +package edu.uc.rphash; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +//import java.util.Arrays; +import java.util.HashMap; +//import java.util.Iterator; +//import java.util.LinkedHashMap; +import java.util.List; +//import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; + +//import org.apache.commons.collections.map.MultiValueMap; +//import org.apache.commons.collections.map.*; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + + + +public class TWRPv6_COV implements Clusterer, Runnable { + + boolean znorm = false; + + private int counter; + private float[] rngvec; + private float[] rngvec2; + private float[] rngvec3; + + private List centroids = null; + + private RPHashObject so; + + public TWRPv6_COV(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + return centroids; + } + + + /* + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + +// this updates the map two cents with different weigths are merged into one. + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, float wcss_1, + float cnt_2, float[] x_2 , float wcss_2) { + + float cnt_r = cnt_1 + cnt_2; + + float[] x_r = new float[x_1.length]; + + float[] var_r1 = new float[x_1.length]; + float[] var_r2 = new float[x_1.length]; + + double var1=0; + double var2=0; + + + for (int i = 0; i < x_1.length; i++) { + x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; + + var_r1[i] = ((-x_r[i] + x_1[i]) * (-x_r[i] + x_1[i]))/(x_r[i]*1000000000); + + var_r2[i] =(((-x_r[i] + x_2[i]) * (-x_r[i] + x_2[i])))/(x_r[i]*1000000000); + + } + + for (int i = 0; i < var_r1.length; i++) { + var1 = var1 + var_r1[i]; + + var2 = var2 + var_r2[i]; + } + double wcsse=0; +// wcsse = ( cnt_1*(wcss_1*wcss_1 + (var1)) + var2 / (cnt_1 + cnt_2 ) ) ; + + wcsse = ( cnt_1*(wcss_1 + (var1)) + var2 / (cnt_1 + cnt_2 ) ) ; + + wcsse = wcsse/(cnt_r); + + // System.out.println("wcsse = " + wcsse); + + float wcss = (float) wcsse; + + float[][] ret = new float[3][]; + ret[0] = new float[1]; + ret[0][0] = cnt_r; + ret[1] = x_r; + ret[2]= new float [1]; + ret[2][0]= wcss; + return ret; + + + } + + + public long hashvec2( float[] xt, float[] x, + HashMap MapOfIDAndCent, HashMap MapOfIDAndCount, int ct, float[] rngvec, HashMap MapOfIDAndWCSS) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) + s= s+1; + + if (MapOfIDAndCent.containsKey(s)) { + + float CurrentCount = MapOfIDAndCount.get(s); + float CurrentCent [] = MapOfIDAndCent.get(s); + float CountForIncomingVector = 1; + float IncomingVector [] = x; + float currentWcss= MapOfIDAndWCSS.get(s); + float incomingWcss= 0; + + + float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, currentWcss, CountForIncomingVector, IncomingVector, incomingWcss ); + + Long UpdatedCount = (long) MergedValues[0][0] ; + + float[] MergedVector = MergedValues[1] ; + + float wcss= MergedValues[2][0]; + + MapOfIDAndCount.put(s , UpdatedCount); + + MapOfIDAndCent.put(s, MergedVector); + + MapOfIDAndWCSS.put(s, wcss); + + + } + + else { + + float[] xlist = x; + MapOfIDAndCent.put(s, xlist); + MapOfIDAndCount.put(s, (long)1); + MapOfIDAndWCSS.put(s, (float)0); + } + } + return s; + } + + + /* + * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid + * vector map + * + * hash the projected vector x and update the hash to centroid and counts + * maps + */ + void addtocounter(float[] x, Projector p, + HashMap IDAndCent,HashMap IDandID,int ct, float[] rngvec , HashMap IDandWCSS) { + float[] xt = p.project(x); + + hashvec2(xt,x,IDAndCent, IDandID, ct,rngvec , IDandWCSS); + } + + + static boolean isPowerOfTwo(long num) { + return (num & -num) == num; + } + + + + /* + * X - data set k - canonical k in k-means l - clustering sub-space Compute + * density mode via iterative deepening hash counting + */ + + + public Multimap findDensityModes2() { + //public Map findDensityModes2() { + HashMap MapOfIDAndCent1 = new HashMap<>(); + HashMap MapOfIDAndCount1 = new HashMap<>(); + HashMap MapOfIDAndWCSS1 = new HashMap<>(); + + HashMap MapOfIDAndCent2 = new HashMap<>(); + HashMap MapOfIDAndCount2 = new HashMap<>(); + HashMap MapOfIDAndWCSS2 = new HashMap<>(); + + HashMap MapOfIDAndCent3 = new HashMap<>(); + HashMap MapOfIDAndCount3 = new HashMap<>(); + HashMap MapOfIDAndWCSS3 = new HashMap<>(); + + HashMap MapOfIDAndCent = new HashMap<>(); + HashMap MapOfIDAndCount = new HashMap<>(); + HashMap MapOfIDAndWCSS = new HashMap<>(); + + + + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + int cutoff = so.getCutoff(); + + int ct = 0; + + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, MapOfIDAndCent1, MapOfIDAndCount1,ct++, rngvec, MapOfIDAndWCSS1); + addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec2,MapOfIDAndWCSS2); + addtocounter(x, projector, MapOfIDAndCent3, MapOfIDAndCount3,ct++, rngvec3,MapOfIDAndWCSS3); + + + } + } + + + float WCSS1 = 0; + float WCSS2 = 0; + float WCSS3 = 0; + + + for (Long cur_id : (MapOfIDAndWCSS1.keySet())) + + { // System.out.println("wcss1 = " + MapOfIDAndWCSS1.get(cur_id)); + WCSS1 = WCSS1 + MapOfIDAndWCSS1.get(cur_id);} + + for (Long cur_id : (MapOfIDAndWCSS2.keySet())) + + { WCSS2 = WCSS2 + MapOfIDAndWCSS2.get(cur_id);} + + for (Long cur_id : (MapOfIDAndWCSS3.keySet())) + + { WCSS3 = WCSS3 + MapOfIDAndWCSS3.get(cur_id);} + +// System.out.println("wcss1 = " + WCSS1); +// System.out.println("wcss2 = " + WCSS2); +// System.out.println("wcss3 = " + WCSS3); + + if ((WCSS1 >= WCSS2) && (WCSS1>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount1; + MapOfIDAndCent = MapOfIDAndCent1; + MapOfIDAndWCSS = MapOfIDAndWCSS1; + System.out.println("winner = tree1"); + } + else if ((WCSS2 >= WCSS1) && (WCSS2>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount2; + MapOfIDAndCent = MapOfIDAndCent2; + MapOfIDAndWCSS = MapOfIDAndWCSS2; + System.out.println("winner = tree2"); + } + else + {MapOfIDAndCount = MapOfIDAndCount3; + MapOfIDAndCent = MapOfIDAndCent3; + MapOfIDAndWCSS = MapOfIDAndWCSS3; + System.out.println("winner = tree3"); + + } + + + System.out.println("NumberOfMicroClustersBeforePruning = "+ MapOfIDAndCent.size()); + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + + + + HashMap denseSetOfIDandCount2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + + MapOfIDAndCent.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + System.out.println("NumberOfMicroClustersAfterPruning&beforesortingLimit = "+ denseSetOfIDandCount2.size()); + + //remove keys with support less than 1 + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(cutoff) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + + + Multimap multimapWeightAndCent = ArrayListMultimap.create(); + + + for (Long keys: sortedIDList2) + + { + + + multimapWeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + + + } + + + return multimapWeightAndCent; + +} + + + public void run() { + rngvec = new float[so.getDimparameter()]; + + rngvec2 = new float[so.getDimparameter()]; + + rngvec3 = new float[so.getDimparameter()]; + + counter = 0; + boolean randVect = so.getRandomVector(); + + // Random r = new Random(so.getRandomSeed()); + // Random r = new Random(3800635955020675334L) ; + Random r = new Random(); + Random r2 = new Random(); + Random r3 = new Random(); + + if (randVect==true){ + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) r.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec2[i] = (float) r2.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec3[i] = (float) r3.nextGaussian(); + + + } else { + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) 0; + } + + + + + Multimap WeightAndClusters = findDensityModes2(); + + + Listcentroids2 = new ArrayList<>(); + List weights2 =new ArrayList<>(); + + + System.out.println("NumberOfMicroClusters_AfterPruning = "+ WeightAndClusters.size()); + System.out.println("getRandomVector = "+ randVect); + + + for (Long weights : WeightAndClusters.keys()) + { + + weights2.add((float)weights); + + } + + + for (Long weight : WeightAndClusters.keySet()) + + { + + centroids2.addAll(WeightAndClusters.get(weight)); + + } + + + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); + + aggloOffline.setWeights(weights2); + + this.centroids = aggloOffline.getCentroids(); + + } + + + public static void main(String[] args) throws FileNotFoundException, + IOException { + + int k = 10;//6; + int d = 200;//16; + int n = 100000; + float var = 1.5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + String Output = "/C:/Users/deysn/Desktop/temp/OutputTwrpCents1" ; + + float f = var; + float avgrealwcss = 0; + float avgtime = 0; + // System.out.printf("%f\t", f); + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); + + // gen.writeCSVToFile(new File("/home/lee/Desktop/reclsh/in.csv")); + + // List data = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" + + RPHashObject o = new SimpleArrayReader(gen.data, k); + + o.setDimparameter(16); + + o.setCutoff(100); + o.setRandomVector(true); + +// System.out.println("cutoff = "+ o.getCutoff()); +// System.out.println("get_random_Vector = "+ o.getRandomVector()); + + TWRPv6_COV rphit = new TWRPv6_COV(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); + + System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } + + //@Override + public void setCutoff(int getCutoff) { + this.so.setCutoff(getCutoff); + } + + //@Override + public void setRandomVector(boolean getRandomVector) { + this.so.setRandomVector(getRandomVector); + } + + +} diff --git a/src/main/java/edu/uc/rphash/TWRPv6_WCSS2.java b/src/main/java/edu/uc/rphash/TWRPv6_WCSS2.java new file mode 100644 index 0000000..be50273 --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRPv6_WCSS2.java @@ -0,0 +1,626 @@ +package edu.uc.rphash; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +//import java.util.Arrays; +import java.util.HashMap; +//import java.util.Iterator; +//import java.util.LinkedHashMap; +import java.util.List; +//import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; + +//import org.apache.commons.collections.map.MultiValueMap; +//import org.apache.commons.collections.map.*; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + + + +public class TWRPv6_WCSS2 implements Clusterer, Runnable { + + boolean znorm = false; + + private int counter; + private float[] rngvec; + private float[] rngvec2; + private float[] rngvec3; + + private List centroids = null; + + private RPHashObject so; + + public TWRPv6_WCSS2(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + return centroids; + } + + + /* + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + +// this updates the map two cents with different weigths are merged into one. + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, float wcss_1, + float cnt_2, float[] x_2 , float wcss_2) { + + float cnt_r = cnt_1 + cnt_2; + + float[] x_r = new float[x_1.length]; + + float[] var_r1 = new float[x_1.length]; + float[] var_r2 = new float[x_1.length]; + + double var1=0; + double var2=0; + + + for (int i = 0; i < x_1.length; i++) { + x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; + + var_r1[i] = ((-x_r[i] + x_1[i]) * (-x_r[i] + x_1[i]))/1000000000; + + var_r2[i] =(((-x_r[i] + x_2[i]) * (-x_r[i] + x_2[i])))/1000000000; + + } + + for (int i = 0; i < var_r1.length; i++) { + var1 = var1 + var_r1[i]; + + var2 = var2 + var_r2[i]; + } + double wcsse=0; + wcsse = ( cnt_1*(wcss_1*wcss_1 + (var1)) + var2 / (cnt_1 + cnt_2 ) ) ; + + // System.out.println("wcsse = " + wcsse); + + float wcss = (float) wcsse; + + float[][] ret = new float[3][]; + ret[0] = new float[1]; + ret[0][0] = cnt_r; + ret[1] = x_r; + ret[2]= new float [1]; + ret[2][0]= wcss; + return ret; + + + } + + + public long hashvec2( float[] xt, float[] x, + HashMap MapOfIDAndCent, HashMap MapOfIDAndCount, int ct, float[] rngvec, HashMap MapOfIDAndWCSS) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) + s= s+1; + + if (MapOfIDAndCent.containsKey(s)) { + + float CurrentCount = MapOfIDAndCount.get(s); + float CurrentCent [] = MapOfIDAndCent.get(s); + float CountForIncomingVector = 1; + float IncomingVector [] = x; + float currentWcss= MapOfIDAndWCSS.get(s); + float incomingWcss= 0; + + + float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, currentWcss, CountForIncomingVector, IncomingVector, incomingWcss ); + + Long UpdatedCount = (long) MergedValues[0][0] ; + + float[] MergedVector = MergedValues[1] ; + + float wcss= MergedValues[2][0]; + + MapOfIDAndCount.put(s , UpdatedCount); + + MapOfIDAndCent.put(s, MergedVector); + + MapOfIDAndWCSS.put(s, wcss); + + + } + + else { + + float[] xlist = x; + MapOfIDAndCent.put(s, xlist); + MapOfIDAndCount.put(s, (long)1); + MapOfIDAndWCSS.put(s, (float)0); + } + } + return s; + } + + + /* + * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid + * vector map + * + * hash the projected vector x and update the hash to centroid and counts + * maps + */ + void addtocounter(float[] x, Projector p, + HashMap IDAndCent,HashMap IDandID,int ct, float[] rngvec , HashMap IDandWCSS) { + float[] xt = p.project(x); + + hashvec2(xt,x,IDAndCent, IDandID, ct,rngvec , IDandWCSS); + } + + + static boolean isPowerOfTwo(long num) { + return (num & -num) == num; + } + + + + /* + * X - data set k - canonical k in k-means l - clustering sub-space Compute + * density mode via iterative deepening hash counting + */ + + + public Multimap findDensityModes2() { + //public Map findDensityModes2() { + HashMap MapOfIDAndCent1 = new HashMap<>(); + HashMap MapOfIDAndCount1 = new HashMap<>(); + HashMap MapOfIDAndWCSS1 = new HashMap<>(); + + HashMap MapOfIDAndCent2 = new HashMap<>(); + HashMap MapOfIDAndCount2 = new HashMap<>(); + HashMap MapOfIDAndWCSS2 = new HashMap<>(); + + HashMap MapOfIDAndCent3 = new HashMap<>(); + HashMap MapOfIDAndCount3 = new HashMap<>(); + HashMap MapOfIDAndWCSS3 = new HashMap<>(); + + + + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + int cutoff = so.getCutoff(); + + int ct = 0; + + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, MapOfIDAndCent1, MapOfIDAndCount1,ct++, rngvec, MapOfIDAndWCSS1); + addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec2,MapOfIDAndWCSS2); + addtocounter(x, projector, MapOfIDAndCent3, MapOfIDAndCount3,ct++, rngvec3,MapOfIDAndWCSS3); + + + } + } + + + + + System.out.println("NumberOfMicroClustersBeforePruning = "+ MapOfIDAndCent1.size()); + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + + + + HashMap denseSetOfIDandCount2_1 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount1.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount1.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount1.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2_1.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + + MapOfIDAndCent1.put(parent_id, new float[]{}); + + // MapOfIDAndCount1.put(parent_id, new Long (0)); + + denseSetOfIDandCount2_1.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2_1.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent1.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2_1.put(cur_id, (long) cur_count); + } + } + } + } + } + + + + + HashMap denseSetOfIDandCount2_2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount2.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount2.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount2.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2_2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + + MapOfIDAndCent2.put(parent_id, new float[]{}); + + // MapOfIDAndCount2.put(parent_id, new Long (0)); + + denseSetOfIDandCount2_2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2_2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent2.put(parent_id, new float[]{}); + // MapOfIDAndCount2.put(parent_id, new Long (0)); + + denseSetOfIDandCount2_2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + + HashMap denseSetOfIDandCount2_3 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount3.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount3.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount3.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2_3.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + + MapOfIDAndCent3.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2_3.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2_3.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent3.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2_3.put(cur_id, (long) cur_count); + } + } + } + } + } + + float WCSS1 = 0; + float WCSS2 = 0; + float WCSS3 = 0; + HashMap denseSetOfIDandCount2 = new HashMap(); + HashMap MapOfIDAndCent = new HashMap<>(); + HashMap MapOfIDAndCount = new HashMap<>(); + HashMap MapOfIDAndWCSS = new HashMap<>(); + + for (Long cur_id : (denseSetOfIDandCount2_1.keySet())) + + { // System.out.println("wcss1 = " + MapOfIDAndWCSS1.get(cur_id)); + WCSS1 = WCSS1 + MapOfIDAndWCSS1.get(cur_id);} + + for (Long cur_id : (denseSetOfIDandCount2_2.keySet())) + + { WCSS2 = WCSS2 + MapOfIDAndWCSS2.get(cur_id);} + + for (Long cur_id : (denseSetOfIDandCount2_3.keySet())) + + { WCSS3 = WCSS3 + MapOfIDAndWCSS3.get(cur_id);} + +// System.out.println("wcss1 = " + WCSS1); +// System.out.println("wcss2 = " + WCSS2); +// System.out.println("wcss3 = " + WCSS3); + + if ((WCSS1 >= WCSS2) && (WCSS1>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount1; + MapOfIDAndCent = MapOfIDAndCent1; + MapOfIDAndWCSS = MapOfIDAndWCSS1; + denseSetOfIDandCount2 = denseSetOfIDandCount2_1; + System.out.println("winner = tree1"); + } + else if ((WCSS2 >= WCSS1) && (WCSS2>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount2; + MapOfIDAndCent = MapOfIDAndCent2; + MapOfIDAndWCSS = MapOfIDAndWCSS2; + denseSetOfIDandCount2 = denseSetOfIDandCount2_2; + System.out.println("winner = tree2"); + } + else + {MapOfIDAndCount = MapOfIDAndCount3; + MapOfIDAndCent = MapOfIDAndCent3; + MapOfIDAndWCSS = MapOfIDAndWCSS3; + denseSetOfIDandCount2 = denseSetOfIDandCount2_3; + System.out.println("winner = tree3"); + + } + + System.out.println("NumberOfMicroClustersAfterPruning&beforesortingLimit = "+ denseSetOfIDandCount2.size()); + + //remove keys with support less than 1 + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(cutoff) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + + + Multimap multimapWeightAndCent = ArrayListMultimap.create(); + + + for (Long keys: sortedIDList2) + + { + + + multimapWeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + + + } + + + return multimapWeightAndCent; + +} + + + public void run() { + rngvec = new float[so.getDimparameter()]; + + rngvec2 = new float[so.getDimparameter()]; + + rngvec3 = new float[so.getDimparameter()]; + + counter = 0; + boolean randVect = so.getRandomVector(); + + // Random r = new Random(so.getRandomSeed()); + // Random r = new Random(3800635955020675334L) ; + Random r = new Random(); + Random r2 = new Random(); + Random r3 = new Random(); + + if (randVect==true){ + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) r.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec2[i] = (float) r2.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec3[i] = (float) r3.nextGaussian(); + + + } else { + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) 0; + } + + + + + Multimap WeightAndClusters = findDensityModes2(); + + + Listcentroids2 = new ArrayList<>(); + List weights2 =new ArrayList<>(); + + + System.out.println("NumberOfMicroClusters_AfterPruning = "+ WeightAndClusters.size()); + System.out.println("getRandomVector = "+ randVect); + + + for (Long weights : WeightAndClusters.keys()) + { + + weights2.add((float)weights); + + } + + + for (Long weight : WeightAndClusters.keySet()) + + { + + centroids2.addAll(WeightAndClusters.get(weight)); + + } + + + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); + + aggloOffline.setWeights(weights2); + + this.centroids = aggloOffline.getCentroids(); + + } + + + public static void main(String[] args) throws FileNotFoundException, + IOException { + + int k = 10;//6; + int d = 200;//16; + int n = 100000; + float var = 1.5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + String Output = "/C:/Users/deysn/Desktop/temp/OutputTwrpCents1" ; + + float f = var; + float avgrealwcss = 0; + float avgtime = 0; + // System.out.printf("%f\t", f); + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); + + // gen.writeCSVToFile(new File("/home/lee/Desktop/reclsh/in.csv")); + + // List data = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" + + RPHashObject o = new SimpleArrayReader(gen.data, k); + + o.setDimparameter(16); + + o.setCutoff(100); + o.setRandomVector(true); + +// System.out.println("cutoff = "+ o.getCutoff()); +// System.out.println("get_random_Vector = "+ o.getRandomVector()); + + TWRPv6_WCSS2 rphit = new TWRPv6_WCSS2(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); + + System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } + + //@Override + public void setCutoff(int getCutoff) { + this.so.setCutoff(getCutoff); + } + + //@Override + public void setRandomVector(boolean getRandomVector) { + this.so.setRandomVector(getRandomVector); + } + + +} diff --git a/src/main/java/edu/uc/rphash/TWRPv6_meanVariance.java b/src/main/java/edu/uc/rphash/TWRPv6_meanVariance.java new file mode 100644 index 0000000..a04f241 --- /dev/null +++ b/src/main/java/edu/uc/rphash/TWRPv6_meanVariance.java @@ -0,0 +1,549 @@ +package edu.uc.rphash; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +//import java.util.Arrays; +import java.util.HashMap; +//import java.util.Iterator; +//import java.util.LinkedHashMap; +import java.util.List; +//import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Stream; + +import edu.uc.rphash.Readers.RPHashObject; +import edu.uc.rphash.Readers.SimpleArrayReader; +import edu.uc.rphash.projections.Projector; +import edu.uc.rphash.tests.StatTests; +import edu.uc.rphash.tests.clusterers.Agglomerative3; +import edu.uc.rphash.tests.generators.GenerateData; +import edu.uc.rphash.util.VectorUtil; + +//import org.apache.commons.collections.map.MultiValueMap; +//import org.apache.commons.collections.map.*; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + + + +public class TWRPv6_meanVariance implements Clusterer, Runnable { + + boolean znorm = false; + + private int counter; + private float[] rngvec; + private float[] rngvec2; + private float[] rngvec3; + + private List centroids = null; + + private RPHashObject so; + + public TWRPv6_meanVariance(RPHashObject so) { + this.so = so; + } + + public List getCentroids(RPHashObject so) { + this.so = so; + return getCentroids(); + } + + @Override + public List getCentroids() { + if (centroids == null) + run(); + return centroids; + } + + + /* + + /* + * X - set of vectors compute the medoid of a vector set + */ + float[] medoid(List X) { + float[] ret = X.get(0); + for (int i = 1; i < X.size(); i++) { + for (int j = 0; j < ret.length; j++) { + ret[j] += X.get(i)[j]; + } + } + for (int j = 0; j < ret.length; j++) { + ret[j] = ret[j] / ((float) X.size()); + } + return ret; + } + +// this updates the map two cents with different weigths are merged into one. + public static float[][] UpdateHashMap(float cnt_1, float[] x_1, float wcss_1, + float cnt_2, float[] x_2 , float wcss_2) { + + float cnt_r = cnt_1 + cnt_2; + + float[] x_r = new float[x_1.length]; + + float[] var_r1 = new float[x_1.length]; + float[] var_r2 = new float[x_1.length]; + + double var1=0; + double var2=0; + + + for (int i = 0; i < x_1.length; i++) { + x_r[i] = (cnt_1 * x_1[i] + cnt_2 * x_2[i]) / cnt_r; + + var_r1[i] = ((-x_r[i] + x_1[i]) * (-x_r[i] + x_1[i]))/1000000000; + + var_r2[i] =(((-x_r[i] + x_2[i]) * (-x_r[i] + x_2[i])))/1000000000; + + } + + for (int i = 0; i < var_r1.length; i++) { + var1 = var1 + var_r1[i]; + + var2 = var2 + var_r2[i]; + } + double wcsse=0; +// wcsse = ( cnt_1*(wcss_1*wcss_1 + (var1)) + var2 / (cnt_1 + cnt_2 ) ) ; + + wcsse = ( cnt_1*(wcss_1 + (var1)) + var2 / (cnt_1 + cnt_2 ) ) ; + + wcsse = wcsse/cnt_r; + + // System.out.println("wcsse = " + wcsse); + + float wcss = (float) wcsse; + + float[][] ret = new float[3][]; + ret[0] = new float[1]; + ret[0][0] = cnt_r; + ret[1] = x_r; + ret[2]= new float [1]; + ret[2][0]= wcss; + return ret; + + + } + + + public long hashvec2( float[] xt, float[] x, + HashMap MapOfIDAndCent, HashMap MapOfIDAndCount, int ct, float[] rngvec, HashMap MapOfIDAndWCSS) { + long s = 1; //fixes leading 0's bug + for (int i = 0; i < xt.length; i++) { +// s <<= 1; + s = s << 1 ; // left shift the bits of s by 1. + if (xt[i] > rngvec[i]) + s= s+1; + + if (MapOfIDAndCent.containsKey(s)) { + + float CurrentCount = MapOfIDAndCount.get(s); + float CurrentCent [] = MapOfIDAndCent.get(s); + float CountForIncomingVector = 1; + float IncomingVector [] = x; + float currentWcss= MapOfIDAndWCSS.get(s); + float incomingWcss= 0; + + + float[][] MergedValues = UpdateHashMap(CurrentCount , CurrentCent, currentWcss, CountForIncomingVector, IncomingVector, incomingWcss ); + + Long UpdatedCount = (long) MergedValues[0][0] ; + + float[] MergedVector = MergedValues[1] ; + + float wcss= MergedValues[2][0]; + + MapOfIDAndCount.put(s , UpdatedCount); + + MapOfIDAndCent.put(s, MergedVector); + + MapOfIDAndWCSS.put(s, wcss); + + + } + + else { + + float[] xlist = x; + MapOfIDAndCent.put(s, xlist); + MapOfIDAndCount.put(s, (long)1); + MapOfIDAndWCSS.put(s, (float)0); + } + } + return s; + } + + + /* + * x - input vector IDAndCount - ID->count map IDAndCent - ID->centroid + * vector map + * + * hash the projected vector x and update the hash to centroid and counts + * maps + */ + void addtocounter(float[] x, Projector p, + HashMap IDAndCent,HashMap IDandID,int ct, float[] rngvec , HashMap IDandWCSS) { + float[] xt = p.project(x); + + hashvec2(xt,x,IDAndCent, IDandID, ct,rngvec , IDandWCSS); + } + + + static boolean isPowerOfTwo(long num) { + return (num & -num) == num; + } + + + + /* + * X - data set k - canonical k in k-means l - clustering sub-space Compute + * density mode via iterative deepening hash counting + */ + + + public Multimap findDensityModes2() { + //public Map findDensityModes2() { + HashMap MapOfIDAndCent1 = new HashMap<>(); + HashMap MapOfIDAndCount1 = new HashMap<>(); + HashMap MapOfIDAndWCSS1 = new HashMap<>(); + + HashMap MapOfIDAndCent2 = new HashMap<>(); + HashMap MapOfIDAndCount2 = new HashMap<>(); + HashMap MapOfIDAndWCSS2 = new HashMap<>(); + + HashMap MapOfIDAndCent3 = new HashMap<>(); + HashMap MapOfIDAndCount3 = new HashMap<>(); + HashMap MapOfIDAndWCSS3 = new HashMap<>(); + + HashMap MapOfIDAndCent = new HashMap<>(); + HashMap MapOfIDAndCount = new HashMap<>(); + HashMap MapOfIDAndWCSS = new HashMap<>(); + + + + // #create projector matrixs + Projector projector = so.getProjectionType(); + projector.setOrigDim(so.getdim()); + projector.setProjectedDim(so.getDimparameter()); + projector.setRandomSeed(so.getRandomSeed()); + projector.init(); + int cutoff = so.getCutoff(); + + int ct = 0; + + { + + for (float[] x : so.getRawData()) + { + addtocounter(x, projector, MapOfIDAndCent1, MapOfIDAndCount1,ct++, rngvec, MapOfIDAndWCSS1); + addtocounter(x, projector, MapOfIDAndCent2, MapOfIDAndCount2,ct++, rngvec2,MapOfIDAndWCSS2); + addtocounter(x, projector, MapOfIDAndCent3, MapOfIDAndCount3,ct++, rngvec3,MapOfIDAndWCSS3); + + + } + } + + + float WCSS1 = 0; + float WCSS2 = 0; + float WCSS3 = 0; + + + for (Long cur_id : (MapOfIDAndWCSS1.keySet())) + + { // System.out.println("wcss1 = " + MapOfIDAndWCSS1.get(cur_id)); + WCSS1 = WCSS1 + MapOfIDAndWCSS1.get(cur_id);} + + for (Long cur_id : (MapOfIDAndWCSS2.keySet())) + + { WCSS2 = WCSS2 + MapOfIDAndWCSS2.get(cur_id);} + + for (Long cur_id : (MapOfIDAndWCSS3.keySet())) + + { WCSS3 = WCSS3 + MapOfIDAndWCSS3.get(cur_id);} + +// System.out.println("wcss1 = " + WCSS1); +// System.out.println("wcss2 = " + WCSS2); +// System.out.println("wcss3 = " + WCSS3); + + if ((WCSS1 >= WCSS2) && (WCSS1>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount1; + MapOfIDAndCent = MapOfIDAndCent1; + MapOfIDAndWCSS = MapOfIDAndWCSS1; + System.out.println("winner = tree1"); + } + else if ((WCSS2 >= WCSS1) && (WCSS2>=WCSS3)) + {MapOfIDAndCount = MapOfIDAndCount2; + MapOfIDAndCent = MapOfIDAndCent2; + MapOfIDAndWCSS = MapOfIDAndWCSS2; + System.out.println("winner = tree2"); + } + else + {MapOfIDAndCount = MapOfIDAndCount3; + MapOfIDAndCent = MapOfIDAndCent3; + MapOfIDAndWCSS = MapOfIDAndWCSS3; + System.out.println("winner = tree3"); + + } + + + System.out.println("NumberOfMicroClustersBeforePruning = "+ MapOfIDAndCent.size()); + + // next we want to prune the tree by parent count comparison + // follows breadthfirst search + + + + HashMap denseSetOfIDandCount2 = new HashMap(); + for (Long cur_id : new TreeSet(MapOfIDAndCount.keySet())) + { + if (cur_id >so.getk()){ + int cur_count = (int) (MapOfIDAndCount.get(cur_id).longValue()); + long parent_id = cur_id>>>1; + int parent_count = (int) (MapOfIDAndCount.get(parent_id).longValue()); + + if(cur_count!=0 && parent_count!=0) + { + if(cur_count == parent_count) { + denseSetOfIDandCount2.put(parent_id, 0L); + // IDAndCent.put(parent_id, new ArrayList<>()); + + MapOfIDAndCent.put(parent_id, new float[]{}); + + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + + } + else + { + if(2 * cur_count > parent_count) { + denseSetOfIDandCount2.remove(parent_id); + + // IDAndCent.put(parent_id, new ArrayList<>()); + MapOfIDAndCent.put(parent_id, new float[]{}); + // MapOfIDAndCount.put(parent_id, new Long (0)); + + denseSetOfIDandCount2.put(cur_id, (long) cur_count); + } + } + } + } + } + + + System.out.println("NumberOfMicroClustersAfterPruning&beforesortingLimit = "+ denseSetOfIDandCount2.size()); + + //remove keys with support less than 1 + Stream> stream2 = denseSetOfIDandCount2.entrySet().stream().filter(p -> p.getValue() > 1); + + + List sortedIDList2= new ArrayList<>(); + // sort and limit the list + stream2.sorted(Entry. comparingByValue().reversed()).limit(cutoff) + .forEachOrdered(x -> sortedIDList2.add(x.getKey())); + + + Multimap multimapWeightAndCent = ArrayListMultimap.create(); + + + for (Long keys: sortedIDList2) + + { + + + multimapWeightAndCent.put((Long)(MapOfIDAndCount.get(keys)), (float[]) (MapOfIDAndCent.get(keys))); + + + } + + + return multimapWeightAndCent; + +} + + + public void run() { + rngvec = new float[so.getDimparameter()]; + + rngvec2 = new float[so.getDimparameter()]; + + rngvec3 = new float[so.getDimparameter()]; + + counter = 0; + boolean randVect = so.getRandomVector(); + + // Random r = new Random(so.getRandomSeed()); + // Random r = new Random(3800635955020675334L) ; + Random r = new Random(); + Random r2 = new Random(); + Random r3 = new Random(); + + if (randVect==true){ + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) r.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec2[i] = (float) r2.nextGaussian(); + + for (int i = 0; i < so.getDimparameter(); i++) + rngvec3[i] = (float) r3.nextGaussian(); + + + } else { + for (int i = 0; i < so.getDimparameter(); i++) + rngvec[i] = (float) 0; + } + + + + + Multimap WeightAndClusters = findDensityModes2(); + + + Listcentroids2 = new ArrayList<>(); + List weights2 =new ArrayList<>(); + + + System.out.println("NumberOfMicroClusters_AfterPruning = "+ WeightAndClusters.size()); + System.out.println("getRandomVector = "+ randVect); + + + for (Long weights : WeightAndClusters.keys()) + { + + weights2.add((float)weights); + + } + + + for (Long weight : WeightAndClusters.keySet()) + + { + + centroids2.addAll(WeightAndClusters.get(weight)); + + } + + + Agglomerative3 aggloOffline = new Agglomerative3(centroids2, so.getk()); + + aggloOffline.setWeights(weights2); + + this.centroids = aggloOffline.getCentroids(); + + } + + + public static void main(String[] args) throws FileNotFoundException, + IOException { + + int k = 10;//6; + int d = 200;//16; + int n = 100000; + float var = 1.5f; + int count = 1; + // System.out.printf("ClusterVar\t"); + // for (int i = 0; i < count; i++) + // System.out.printf("Trial%d\t", i); + // System.out.printf("RealWCSS\n"); + + String Output = "/C:/Users/deysn/Desktop/temp/OutputTwrpCents1" ; + + float f = var; + float avgrealwcss = 0; + float avgtime = 0; + // System.out.printf("%f\t", f); + GenerateData gen = new GenerateData(k, n/k, d, f, true, .5f); + + // gen.writeCSVToFile(new File("/home/lee/Desktop/reclsh/in.csv")); + + // List data = "/C:/Users/user/Desktop/temp/OutputTwrpCents1" + + RPHashObject o = new SimpleArrayReader(gen.data, k); + + o.setDimparameter(16); + + o.setCutoff(100); + o.setRandomVector(true); + +// System.out.println("cutoff = "+ o.getCutoff()); +// System.out.println("get_random_Vector = "+ o.getRandomVector()); + + TWRPv6_meanVariance rphit = new TWRPv6_meanVariance(o); + long startTime = System.nanoTime(); + List centsr = rphit.getCentroids(); + + avgtime += (System.nanoTime() - startTime) / 100000000; + + avgrealwcss += StatTests.WCSSEFloatCentroid(gen.getMedoids(), + gen.getData()); + + VectorUtil.writeCentroidsToFile(new File(Output),centsr, false); + + System.out.printf("%.0f\t", StatTests.WCSSECentroidsFloat(centsr, gen.data)); + System.gc(); + + System.out.printf("%.0f\n", avgrealwcss / count); + + + } + + @Override + public RPHashObject getParam() { + return so; + } + + @Override + public void setWeights(List counts) { + // TODO Auto-generated method stub + + } + + @Override + public void setData(List centroids) { + this.centroids = centroids; + + } + + @Override + public void setRawData(List centroids) { + if (this.centroids == null) + this.centroids = new ArrayList<>(centroids.size()); + for (float[] f : centroids) { + this.centroids.add(new Centroid(f, 0)); + } + } + + @Override + public void setK(int getk) { + this.so.setK(getk); + } + + @Override + public void reset(int randomseed) { + centroids = null; + so.setRandomSeed(randomseed); + } + + @Override + public boolean setMultiRun(int runs) { + return false; + } + + //@Override + public void setCutoff(int getCutoff) { + this.so.setCutoff(getCutoff); + } + + //@Override + public void setRandomVector(boolean getRandomVector) { + this.so.setRandomVector(getRandomVector); + } + + +}