diff --git a/jars/sqlite-jdbc-3.30.1.jar b/jars/sqlite-jdbc-3.30.1.jar new file mode 100644 index 00000000..85837370 Binary files /dev/null and b/jars/sqlite-jdbc-3.30.1.jar differ diff --git a/src/test/Database_Init_Del.java b/src/test/Database_Init_Del.java new file mode 100644 index 00000000..b04b5bcc --- /dev/null +++ b/src/test/Database_Init_Del.java @@ -0,0 +1,130 @@ +package test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class Database_Init_Del { + + public Database_Init_Del() { + } + + public static void connect(String dbname) { + Connection conn = null; + try { + String url = "jdbc:sqlite:"+dbname; + // create a connection to the database + conn = DriverManager.getConnection(url); + + System.out.println("Connection to SQLite has been established."); + + String query="PRAGMA foreign_keys = ON"; + Statement lock=conn.createStatement(); + lock.execute(query); + createILtable(lock); + createVLtable(lock); + createFVtable(lock); + createFNtable(lock); + createCLtable(lock); + createVtable(lock); + } catch (SQLException e) { + System.out.println(e.getMessage()); + } finally { + try { + if (conn != null) { + conn.close(); + System.out.println("Connection closed"); + } + } catch (SQLException ex) { + System.out.println(ex.getMessage()); + } + } + } + + private static void createVtable(Statement lock) throws SQLException { + String query="DROP TABLE IF EXISTS `vectors`"; + lock.execute(query); + query="CREATE TABLE `vectors` (\r\n" + + " `v_id` INTEGER PRIMARY KEY, \r\n" + + " `image_name` TEXT \r\n" + + ");"; + lock.execute(query); + System.out.println("VECTORS created"); + } + + private static void createILtable(Statement lock) throws SQLException { + String query="DROP TABLE IF EXISTS `image_list`"; + + lock.execute(query); + query="CREATE TABLE `image_list` (\r\n" + + " `img_id` INTEGER PRIMARY KEY, \r\n" + + " `image_name` TEXT, \r\n" + + " `instance_type` TEXT, \r\n" + + " `cell_type` TEXT, \r\n" + + " `v_id` INTEGER, \r\n" + + " FOREIGN KEY(v_id) REFERENCES vectors(v_id)\r\n" + + ");"; + + lock.execute(query); + System.out.println("IMAGE_LIST created"); + } + + private static void createVLtable(Statement lock) throws SQLException { + String query="DROP TABLE IF EXISTS `vector_list`"; + lock.execute(query); + query="CREATE TABLE `vector_list` (\r\n" + + " `vl_id` INTEGER PRIMARY KEY, \r\n" + + " `v_id` INTEGER, \r\n" + + " `value` REAL, \r\n" + + " FOREIGN KEY(v_id) REFERENCES vectors(v_id)\r\n" + + ")"; + lock.execute(query); + System.out.println("VECTOR_LIST created"); + } + + private static void createCLtable(Statement lock) throws SQLException { + String query="DROP TABLE IF EXISTS `class_list`"; + lock.execute(query); + query="CREATE TABLE `class_list` (\r\n" + + " `cl_id` INTEGER PRIMARY KEY,\r\n" + + " `cl_name` TEXT \r\n" + + ")"; + lock.execute(query); + System.out.println("CLASS_LIST created"); + } + + private static void createFNtable(Statement lock) throws SQLException { + String query="DROP TABLE IF EXISTS `features_names`"; + lock.execute(query); + query="CREATE TABLE `features_names` (\r\n" + + " `fn_id` INTEGER PRIMARY KEY, \r\n" + + " `f_name` TEXT \r\n" + + ")"; + lock.execute(query); + System.out.println("FEATURES_NAMES created"); + } + + private static void createFVtable(Statement lock) throws SQLException { + String query="DROP TABLE IF EXISTS `features_values`" ; + lock.execute(query); + query="CREATE TABLE `features_values` (\r\n" + + " `f_id` INTEGER PRIMARY KEY, \r\n" + + " `f_value` REAL, \r\n" + + " `fn_id` INTEGER, \r\n" + + " `cl_id` INTEGER, \r\n" + + "FOREIGN KEY(fn_id) REFERENCES features_names(fn_id), \r\n" + + "FOREIGN KEY(cl_id) REFERENCES class_list(cl_id) \r\n" + + ")"; + + lock.execute(query); + System.out.println("FEATURES_VALUES created"); + } + + public static void main(String[] args) { + String path="C:\\Users\\billa\\Documents\\GitHub\\ACTIVESEGMENTATION\\classif.db"; + connect(path); + } + +} + diff --git a/src/test/TestOnlineCells.java b/src/test/TestOnlineCells.java index e17ab141..1548d0c7 100644 --- a/src/test/TestOnlineCells.java +++ b/src/test/TestOnlineCells.java @@ -1,26 +1,37 @@ package test; import java.io.File; +import java.sql.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; +import java.util.Hashtable; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.StringTokenizer; +import java.util.Arrays; import activeSegmentation.ASCommon; import activeSegmentation.moment.Haralick_feature_; import activeSegmentation.moment.IJstat_feature_; import activeSegmentation.moment.Legendre_feature_; import activeSegmentation.moment.Zernike_feature_; +import activeSegmentation.learning.ClassifierManager; import ij.IJ; import ij.ImageJ; import ij.ImagePlus; import ij.gui.Roi; +import ij.measure.Calibration; +import ij.measure.ResultsTable; +import ij.plugin.filter.Analyzer; import ij.process.ImageProcessor; +import ijaux.TestUtil; import ijaux.datatype.Pair; import weka.classifiers.Evaluation; import weka.classifiers.functions.SMO; @@ -29,9 +40,12 @@ import weka.core.DenseInstance; import weka.core.Instance; import weka.core.Instances; +import weka.core.Utils; import weka.filters.Filter; import weka.filters.unsupervised.attribute.Normalize; + + public class TestOnlineCells { //features Legendre_feature_ legendre_filter_=new Legendre_feature_(); @@ -47,6 +61,14 @@ public class TestOnlineCells { SMO randomForest= new SMO(); //training Data Instances trainingData; + public final static String pluginname="SQLite client"; + public final static String version="3.36.0.3"; + public final static String driver="org.sqlite.JDBC"; + public final String micro = "\u00B5"; + + private Connection con; + private int vid=1; + public TestOnlineCells() { // TODO Auto-generated constructor stub @@ -57,91 +79,88 @@ public TestOnlineCells() { featureNames.put(glcm_Filter_.getKey(), glcm_Filter_.getFeatureNames()); featureNames.put(zernike_Filter_.getKey(), zernike_Filter_.getFeatureNames()); featureNames.put(imageJfeatures.getKey(), imageJfeatures.getFeatureNames()); - + } public static void main(String[] args) throws Exception { // TODO Auto-generated method stub new ImageJ(); TestOnlineCells test= new TestOnlineCells(); - String directory="C:\\Users\\vohra\\Documents\\EM\\hela_test\\"; + String directory="C:\\Users\\billa\\Documents\\hela_test2\\"; test.run(directory); - //System.out.println(dirs); } public void run(String directory) throws Exception { Map>cells=loadCells(directory); List trainInstances= new ArrayList<>(); List testInstances= new ArrayList<>(); - // cross-validation step for(String key: cells.keySet()) { + //System.out.println(key); List cellsinside=cells.get(key); Collections.shuffle(cellsinside); - for(CellType cell: cellsinside.subList(0, cellsinside.size()*4/5)) { + for (CellType cell: cellsinside.subList(0, cellsinside.size()*4/5)) { trainInstances.add(cell.getInstances()); } - for(CellType cell: cellsinside.subList(cellsinside.size()*4/5,cellsinside.size())) { + for (CellType cell: cellsinside.subList(cellsinside.size()*4/5,cellsinside.size())) { testInstances.add(cell.getInstances()); } - } - Collections.shuffle(trainInstances); - - - trainClassfier(trainInstances, testInstances); - + } + trainClassfier(trainInstances, testInstances,cells); + //Collections.shuffle(trainInstances); } - - public void trainClassfier(List trainInstances, List testInstances) throws Exception { - + + public void trainClassfier(List trainInstances, List testInstances,Map> cells) throws Exception{ + List trainInstances2=trainInstances; + Collections.shuffle(trainInstances); // List trainInstances=instances.subList(0, instances.size()*4/5); // List testInstances=instances.subList(instances.size()*4/5, instances.size() ); - - Instances testData = new Instances(ASCommon.INSTANCE_NAME, attributes, 1 ); - testData.setClassIndex(numberOfFeatures-1); - for(Instance instance:testInstances) { - testData.add(instance); - } - trainingData = new Instances(ASCommon.INSTANCE_NAME, attributes, 1 ); - trainingData.setClassIndex(numberOfFeatures-1); - for(Instance instance:trainInstances) { - trainingData.add(instance); - } - System.out.println(trainingData); - //smo.setBatchSize("100"); - // randomForest.setNumIterations(100); + + Instances testData = new Instances(ASCommon.INSTANCE_NAME, attributes, 1 ); + testData.setClassIndex(numberOfFeatures-1); + for(Instance instance:testInstances) { + testData.add(instance); + } + trainingData = new Instances(ASCommon.INSTANCE_NAME, attributes, 1 ); + trainingData.setClassIndex(numberOfFeatures-1); + for(Instance instance:trainInstances) { + trainingData.add(instance); + } + //smo.setBatchSize("100"); + // randomForest.setNumIterations(100); try { - ; Instances newdata = trainingData; randomForest.buildClassifier(newdata); System.out.println(randomForest.toString()); Evaluation eval = new Evaluation(newdata); //eval.evaluateModel(smo, test); + eval.crossValidateModel(randomForest, newdata, 10, new Random(1)); System.out.println(eval.toSummaryString("\nResults\n======\n", false)); - System.out.println(eval.toClassDetailsString()); - Evaluation eval1 = new Evaluation(newdata); - Instances testNewData = testData; - eval1.evaluateModel(randomForest, testNewData); - System.out.println(eval1.toSummaryString("\nResults\n======\n", false)); - System.out.println(eval1.toClassDetailsString()); + System.out.println(eval.toClassDetailsString()); + startDB(trainingData,testData,trainInstances2,testInstances,cells,randomForest); + + //Evaluation eval1 = new Evaluation(newdata); + //Instances testNewData = testData; + //eval1.evaluateModel(randomForest, testNewData); + //System.out.println(eval1.toSummaryString("\nResults\n======\n", false)); + //System.out.println(eval1.toClassDetailsString()); }catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } - + } - - + public Map> loadCells(String directory){ Map> cells= new HashMap<>(); Set dirs=loadDirectory(directory); for(String cellType: dirs) { classlabels.add(cellType); } - createHeader(classlabels); + createHeader(classlabels); Map classIndexMap=new HashMap<>(); int index=0; for(String label: classlabels) { @@ -161,12 +180,210 @@ public Map> loadCells(String directory){ currentList.add(cell); cells.put(cellType, currentList); System.out.println(tiffile); + } //System.out.println(cellsDir); } - //System.out.println(cells); return cells; } + + int c=0; + + boolean connStart(String dbName) { + //creates connection with the database + String dbUrl="jdbc:sqlite:"+ dbName; + try { + //Class.forName("org.gjt.mm.mysql.Driver"); + Class.forName(driver); + } catch(Exception ex) { + IJ.log("Can't find Database driver class: " + ex); + return false; + } + try { + con = DriverManager.getConnection(dbUrl); + IJ.log("Connected to " + dbUrl); + return true; + } catch(SQLException ex) { + IJ.log("SQLException: " + ex); + return false; + } + } + + private void startDB(Instances trainingData2, Instances testData,List trainInstances,List testInstances,Map> cells,SMO randomforest) { + try { + //starts the implementation process + trainingData2.addAll(testData); + Enumeration enums=trainingData2.enumerateInstances(); + connStart("C:\\Users\\billa\\Documents\\GitHub\\ACTIVESEGMENTATION\\classif.db"); + insert(enums,trainInstances,testInstances,cells,randomforest); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + + public List shuffle4same(List trainInstances,List tifs,List ct,List toi){ + //shuffle 4 arrays the same way + double random_number = Math.floor(Math.random() * 10); + Collections.shuffle(trainInstances,new Random((int)random_number)); + Collections.shuffle(tifs.subList(0,tifs.size()*4/5),new Random((int)random_number)); + Collections.shuffle(ct.subList(0,ct.size()*4/5),new Random((int)random_number)); + Collections.shuffle(toi.subList(0,toi.size()*4/5),new Random((int)random_number)); + List all = new ArrayList<>(); + //gets a final array that contains tif images,cell types, type of instances + for(int i = 0; i < tifs.size(); i++) + { + all.add(tifs.get(i)); + all.add(ct.get(i)); + all.add(toi.get(i)); + } + //it is like [tif1,ct1,toi1,tif2,ct2,toi2,...] + return all; + } + + public void insert(Enumeration enums, List trainInstances,List testInstances,Map> cells,SMO randomforest) throws Exception { + List tifs=new ArrayList<>(); + List ct=new ArrayList<>(); + List toi=new ArrayList<>(); + for(String key: cells.keySet()) { + List cellsinside=cells.get(key); + Collections.shuffle(cellsinside); + for (CellType cell: cellsinside.subList(0, cellsinside.size()*4/5)) { + tifs.add(cell.getTiffile().replace("C:\\Users\\billa\\Documents\\hela_test2\\"+cell.getCelltype()+"\\","")); + ct.add(cell.getCelltype()); + toi.add("Train"); + } + for (CellType cell: cellsinside.subList(cellsinside.size()*4/5,cellsinside.size())) { + tifs.add(cell.getTiffile().replace("C:\\Users\\billa\\Documents\\hela_test2\\"+cell.getCelltype()+"\\","")); + ct.add(cell.getCelltype()); + toi.add("Test"); + } + } + List all=shuffle4same(trainInstances,tifs,ct,toi); + //shuffle them the same way for training + double [][][] w=randomforest.sparseWeights();//contains the attribute weights + String [][][] nm=randomforest.attributeNames();//contains the attribute names + while (enums.hasMoreElements()) { + Instance currentList=enums.nextElement();//trainingData.instance(0); + try { + double[] probv=randomforest.distributionForInstance(currentList); + TestUtil.printvector(probv); + + double ss=0; + for (double s: probv) { + ss+=s; + } + System.out.println("\n"+ ss); //prints out that the sum of the probability values are 1 + + //insert to image_list the name of the image,the instance type,the cell type and the id of the vector + String update="INSERT INTO image_list (image_name, instance_type, cell_type,v_id) " + + "VALUES ( ?, ?,?,?)"; + + PreparedStatement ips=con.prepareStatement(update); + ips.setString(1,all.get(c)); + ips.setString(3,all.get(c+1)); + ips.setString(2,all.get(c+2)); + ips.setInt(4, vid); + ips.executeUpdate(); + //inserts to vectors list the image names + update="INSERT INTO vectors ( image_name) " + + "VALUES ( ?)"; + + PreparedStatement vps = con.prepareStatement(update); + vps.setString(1, all.get(c)); + vps.executeUpdate(); + //inserts to the vector_list the vector id and the probability values for this vector id(10 for each) + update="INSERT INTO vector_list ( v_id, value) VALUES ( ?, ?)"; + // for looping into the prob vectors + PreparedStatement vlps = con.prepareStatement(update); + + for(double vec:probv) { + vlps.setInt(1, vid); + vlps.setDouble(2, vec); + vlps.executeUpdate(); + } + ips.clearParameters(); + vps.clearParameters(); + vlps.clearParameters(); + ips.close(); + vps.close(); + vlps.close(); + vid++; + c=c+3; + //do it once + if(c==3) { + //inserts in the class list the class names + update="INSERT INTO class_list (cl_name) " + + "VALUES (?)"; + PreparedStatement clps=con.prepareStatement(update); + for(int cl = 0; cl < classlabels.size(); cl++) { + clps.setString(1,classlabels.get(cl)); + clps.executeUpdate(); + } + clps.clearParameters(); + clps.close(); + //inserts into the features_names list the names of each feature + update="INSERT INTO features_names (f_name) " + + "VALUES (?)"; + PreparedStatement fnps=con.prepareStatement(update); + String[] max={""}; + //below we get the max features that may be presented for each vector + for(String [][] array2d : nm){ + for(String[] array : array2d){ + if(array!=null) { + if(array.length>max.length) { + max=array; + } + } + } + } + for(int f = 0; f < max.length; f++) { + fnps.setString(1,max[f]); + fnps.executeUpdate(); + } + fnps.clearParameters(); + fnps.close(); + //insterts into the features_values the value of each feature along with + // the feature name id and the class id + update="INSERT INTO features_values (f_value,fn_id,cl_id) " + + "VALUES (?,?,?)"; + PreparedStatement fps=con.prepareStatement(update); + int cl=1; + for (double[][] array_2D: w) { + outerLoop: + for (double[] array_1D: array_2D) { + int fn=1; + if(array_1D!=null) { + for(double elem:array_1D) { + fps.setDouble(1,elem); + fps.setInt(2,fn); + fps.setInt(3,cl); + fps.executeUpdate(); + fn++; + } + }else { + fps.setInt(1,0); + fps.setInt(2,0); + fps.setInt(3,cl); + fps.executeUpdate(); + continue outerLoop; + } + } + cl++; + } + fps.clearParameters(); + fps.close(); + } + } catch (SQLException E) { + IJ.log("SQL message: " + E.getMessage()); + + } + } + System.out.println("DONE"); + con.close(); + } + public void createHeader(List classlabels) { int featureSIndex=0; @@ -245,4 +462,4 @@ public Set loadDirectory(String directory){ } -} +} \ No newline at end of file