Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions Planter.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
warnings.filterwarnings('ignore')

# =================== import Planter src ===================
# funtions
# functions
from src.functions.figure_to_ASCII import *
from src.functions.json_encoder import *
from src.functions.project_tree_graph import *
Expand All @@ -45,12 +45,12 @@ def Planter(iteration = 0):

# =================== set argparse for Planter ===================

parser = argparse.ArgumentParser(prog="Please use commend '-h' or '--help' for further information",
parser = argparse.ArgumentParser(prog="Please use command '-h' or '--help' for further information",
usage='Planter can embed several ML algorithms to programmable network devices',
description='It is a on going work, if you find any bugs, please feel free to contact changgang.zheng@eng.ox.ac.uk, it is really important to us. Thank you.',
epilog='Play happy with Planter ~')
parser.add_argument("-m", "--manually_config", help="Manually config Planter or not? If not set, please config the following file <src/config/Planter_config.json>. If set, the Planter will ask you to input all the necessary configs.", action="store_true")
parser.add_argument("-o", "--auto_optimisation", help="Use auto hyperparameters tunning or not? If set, the Planter will trigger auto optimisation process. Please do not use this mode with other modes, Planter will handle everything.", action="store_true")
parser.add_argument("-o", "--auto_optimisation", help="Use auto hyperparameters tuning or not? If set, the Planter will trigger auto optimisation process. Please do not use this mode with other modes, Planter will handle everything.", action="store_true")
parser.add_argument("-t", "--testing_mode", help="Use testing mode or not? If set, the testing mode will only use at most 20000 input data and 5000 testing data.", action="store_true")
parser.add_argument("-d", "--draw_project_tree", help="Draw the project tree or not? If set, this mode will output a tree graph of the Planter project.", action="store_true")
parser.add_argument("-f", "--draw_function_call_graph", help="Draw function calling dependency graph or not, if set, after running Planter, a function calling dependency graph will be generated under <src/logs>.", action="store_true")
Expand Down Expand Up @@ -147,19 +147,19 @@ def Planter(iteration = 0):
round = 0
while True:
round += 1
# call the function to find best paraemters and generate the config file
# call the function to find best parameters and generate the config file
main_functions.config_generator(args, round)
# call Plantrer to verify the generated configs
print('= Begin varifcation for the generated configs. Rounds', round)
# call Planter to verify the generated configs
print('= Begin verification for the generated configs. Rounds', round)
args.manually_config = False
Planter(round)
# reload the config file
Planter_config = reload_config('src/configs/Planter_config.json')
if Planter_config['optimisation config']['compile result'] == 'y':
print('= Verification seccess. Data plane and control plane codes are generated with best hyperparameters.')
print('= Verification success. Data plane and control plane codes are generated with best hyperparameters.')
break
else:
print('= Verification fales. Regenerate configs with higher constrained configs')
print('= Verification fails. Regenerate configs with higher constrained configs')
exit()
# =================== auto config mode config and operations finished ===================
# print('= args:', args)
Expand Down
8 changes: 3 additions & 5 deletions src/configs/requirements_pip3.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,16 @@ scipy==1.5.2
joblib==1.0.0
pydotplus==2.0.2
pandas==1.1.3
xgboost==0.90
xgboost>=1.5.0
matplotlib==3.3.2
torch==1.4.0
numpy==1.19.2
numpy>=1.21.0
category_encoders==2.2.2
scikit_learn
scikit-learn
packaging==21.0
jsonschema==3.2.0
scapy==2.4.3
seaborn==0.11.2
category_encoders==2.2.2
wget==3.2
pytorch
ipython
tqdm
8 changes: 4 additions & 4 deletions src/configs/requirements_sudo_pip3.txt
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
numpy==1.19.2
scikit_learn==1.0
numpy>=1.21.0
scikit-learn==1.0
scapy==2.4.3
seaborn==0.11.2
category_encoders==2.2.2
wget==1.20.1
pytorch==0.4.0
multiprocess=0.70.12
torch
multiprocess==0.70.12
4 changes: 2 additions & 2 deletions src/functions/Range_to_LPM.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,11 +193,11 @@ def Table_to_sep_val(table):
lasat_lable = table[dict]
if dict == 0:
Table[dict] = table[dict] # When you get to the end of the for loop, the last value in table and final key is 0
#This for-loop selects all feature values at bounderies and puts them in Table
#This for-loop selects all feature values at boundaries and puts them in Table
#For instance if the first 20 actions of the exact match table were 112111112111211121111121 then Table would be 1121212121 in reverse order
#Selects the value before the value change

separation_value = list(Table.keys()) #Selects index of seperation values (starting from highest e.g 99,98,86,77,56,29,6,1)
separation_value = list(Table.keys()) #Selects index of separation values (starting from highest e.g 99,98,86,77,56,29,6,1)

return separation_value, separation_value_input

Expand Down
2 changes: 1 addition & 1 deletion src/help/Adding_Your_Own_Design/Adding_model.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ In ```table_generator.py```, there are two compulsory functions to be written, `
```
sklearn_test_y # data frame/ndarray/list
# sklearn (baseline) inference result (output labels)
# generated by funtion test_tables(*)
# generated by function test_tables(*)

test_X # data frame/ndarray/list
# testing data
Expand Down
2 changes: 1 addition & 1 deletion src/help/Common_Issues/Envronment_Issues.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

If it turns out that some packages were not installed (even though you have set up the environment based on ```requirements.txt```), install the missing packages manually.

💡 To set up Planter on P4Pi-enabled BMv2, follow the wiki page in [this link](./src/help/Throughput_Test/Planter_throughput_test_P4Pi-BMv2.md).
💡 To set up Planter on P4Pi-enabled BMv2, follow the wiki page in [this link](../Throughput_Test/Planter_throughput_test_P4Pi-BMv2.md).


## Common Running Issues:
Expand Down
2 changes: 1 addition & 1 deletion src/help/Planter_Supports/Planter_Supported_Datasets.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Planter currently supports 7 datasets.

**4. AWID3** [Link](https://icsdweb.aegean.gr/awid/awid3) - The AWID3 dataset is a network intrusion detection dataset with 13 different attacks. The data is collected from 16 different physical devices and VMs.

**5. Requet** [Link](https://github.com/Wimnet/RequetDataSet) - Requet is a Quality of Experience (QoE) metric detection dataset collected from video streaming services to give the records of video playback information.
**5. Request** [Link](https://github.com/Wimnet/RequetDataSet) - Request is a Quality of Experience (QoE) metric detection dataset collected from video streaming services to give the records of video playback information.

**6. Jane Street Market** [Link](https://www.kaggle.com/c/jane-street-market-prediction) - Jane Street Markets is a high-frequency trading dataset with 500 days of historical data. Specifically, it contains 130 features.

Expand Down
2 changes: 1 addition & 1 deletion src/models/Autoencoder/Type_1/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for Autoencoder. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for Autoencoder. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
2 changes: 1 addition & 1 deletion src/models/Autoencoder/Type_LB/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for Autoencoder. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for Autoencoder. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
2 changes: 1 addition & 1 deletion src/models/Bayes/Type_1/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
2 changes: 1 addition & 1 deletion src/models/Bayes/Type_2/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
2 changes: 1 addition & 1 deletion src/models/Bayes/Type_3/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
2 changes: 1 addition & 1 deletion src/models/Bayes/Type_LB/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
2 changes: 1 addition & 1 deletion src/models/Bayes/Type_LB_Bernoulli/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for Bayes. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
2 changes: 1 addition & 1 deletion src/models/DT/Type_1/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for DT. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for DT. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
28 changes: 14 additions & 14 deletions src/models/DT/Type_1/table_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def recurse(left, right, child, lineage=None):
else:
sign = g
clause = clause + i[3] + sign + str(i[2]) + ' and '
# wirte the node information into text file
# write the node information into text file
a = list(value[node][0])
ind = a.index(np.max(a))
clause = clause[:-4] + ' then ' + str(ind)
Expand Down Expand Up @@ -164,11 +164,11 @@ def generate_feature_tables(split, num_features,feature_max, table):
for i in range(num_features):
table["feature "+str(i)] = {}
count_code = 0
nife = sorted(split["feature "+str(i)])
knife = sorted(split["feature "+str(i)])
for j in range(feature_max[i]+1):
if nife !=[] :
if len(nife) > count_code:
if j-1 == nife[count_code]:
if knife !=[] :
if len(knife) > count_code:
if j-1 == knife[count_code]:
count_code+=1
table["feature " + str(i)][j] = count_code
return table
Expand All @@ -195,7 +195,7 @@ def find_classification(textfile, feature_split, num_features):
sign.append(re.findall(r"(<=|>)", line))
num.append(re.findall(r"\d+\.?\d*", line))
f.close()
classfication = []
classification = []
featuren = {}
for i in range(len(fea)):
for l in range(num_features):
Expand All @@ -220,16 +220,16 @@ def find_classification(textfile, feature_split, num_features):
for l in range(num_features):
feature_n[l].append(featuren[l])
a = len(num[i])
classfication.append(num[i][a - 1])
classification.append(num[i][a - 1])

return feature_n, classfication
return feature_n, classification


def find_path_for_leaf_nodes(feature_n, classfication, num_features):
def find_path_for_leaf_nodes(feature_n, classification, num_features):
path_to_leaf = {}
for i in range(len(classfication)):
for i in range(len(classification)):
path_to_leaf["path "+str(i)] = {}
path_to_leaf["path " + str(i)]["leaf"] = classfication[i]
path_to_leaf["path " + str(i)]["leaf"] = classification[i]
for j in range(num_features):
path_to_leaf["path " + str(i)]["feature "+str(j)] = feature_n[j][i]
return path_to_leaf
Expand Down Expand Up @@ -266,8 +266,8 @@ def generate_table(model, tree_index, num_features, g_table, feature_max):
g_table[tree_index] = {}
g_table[tree_index] = generate_feature_tables(feature_split, num_features, feature_max, g_table[tree_index])

feature_n, classfication = find_classification(textfile, feature_split , num_features)
path_to_leaf = find_path_for_leaf_nodes(feature_n, classfication, num_features)
feature_n, classification = find_classification(textfile, feature_split , num_features)
path_to_leaf = find_path_for_leaf_nodes(feature_n, classification, num_features)
code_width_for_feature = np.zeros(num_features)
for i in range(num_features):
code_width_for_feature[i] = int(np.ceil(math.log(g_table[tree_index]['feature ' + str(i)][np.max(list(g_table[tree_index]['feature ' + str(i)].keys()))]+1,2))) or 1
Expand Down Expand Up @@ -383,7 +383,7 @@ def test_tables(sklearn_test_y, test_X, test_y):
Ternary_Table = json.load(open('Tables/Ternary_Table.json', 'r'))
Exact_Table = json.load(open('Tables/Exact_Table.json', 'r'))

print('Test the exact feature table, extact code and decision table (feel free if the acc to sklearn is slightly lower than 1)')
print('Test the exact feature table, exact code and decision table (feel free if the acc to sklearn is slightly lower than 1)')
same = 0
correct = 0
error = 0
Expand Down
2 changes: 1 addition & 1 deletion src/models/DT/Type_1_xsa/readme.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
This folder contains Planter-supported ML modules (training, algortihm mapping, and ML-related P4 generation) for DT. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
This folder contains Planter-supported ML modules (training, algorithm mapping, and ML-related P4 generation) for DT. ```dedicated_p4.py``` generates the P4 code dedicated to this ML model and ```table_generator.py``` generate ML model parameters in the format of table enries. Please refer to ```./Docs/Planter_User_Document.pdf```for further information.
Loading