diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e2638f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.pyc +__pycache__/ +wandb/ + diff --git a/3DOrangeTree/README.md b/3DOrangeTree/README.md deleted file mode 100644 index da39147..0000000 --- a/3DOrangeTree/README.md +++ /dev/null @@ -1,52 +0,0 @@ - - - -# ConSpec on the OrangeTree task - -## Information -The 3D environments were designed and built in SilicoLabs Experimenter software (https://www.silicolabs.ca/), which is built on top of Unity. - -The ConSpec code is built on top of the basic PPO code taken from the repository: https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail - -The interface between the environment and Pytorch was built on top of ML-agents: https://github.com/Unity-Technologies/ml-agents - - -## Installation -There are 3 environment binary files that need to be -1) downloaded from the Google Drive https://drive.google.com/drive/folders/1azC6fJoQjth_rcDm499HfetS7AU9vUtB?usp=sharing -2) unzipped. - -To create this environment, install the packages in the requirements.txt file -In main.py --> modify the directory line 52. The directory line should where you placed the environment files. - - -### Launching - -#### To run ConSpec on the OrangeTree task: -In the command line, first access the environment by running: -``` -chmod -R 755 /home/chen/PycharmProjects/ProjTreeSimplePink2extrahardLinux4590_338084842miss/GridWorld.x86_64 -``` -Then run: -``` -python main.py --algo Conspec --use-gae --lr 2e-4 --clip-param 0.08 --value-loss-coef 0.5 --num-processes 16 --num-steps 65 --num-mini-batch 4 --log-interval 1 --use-linear-lr-decay --entropy-coef 0.02 --lrCL 20e-4 --choiceCLparams 0 --seed 80000 --head 8 --factorR 0.5 -``` - -#### To test the runs from main.py on the modified OrangeTree task with novel black objects: -In the command line, first access the environment by running: -``` -chmod -R 755 /home/chen/PycharmProjects/ProjTreeSimplePink2extrahardLinux4590_33_84842onevarb/GridWorld.x86_64 -``` -Then run: -``` -python maintestBlackobj.py --algo Conspec --use-gae --lr 2e-4 --clip-param 0.08 --value-loss-coef 0.5 --num-processes 16 --num-steps 65 --num-mini-batch 4 --log-interval 1 --use-linear-lr-decay --entropy-coef 0.02 --lrCL 20e-4 --choiceCLparams 0 --seed 80000 --head 8 --factorR 0.5 -``` -#### To test the runs from main.py on the new green and pink environment: -In the command line, first access the environment by running: -``` -chmod -R 755 /home/chen/PycharmProjects/ProjTreeSimplePink2extrahardLinux4590_338084842TestmissRoommgp/GridWorld.x86_64 -``` -Then run: -``` -python maintestNewContext.py --algo Conspec --use-gae --lr 2e-4 --clip-param 0.08 --value-loss-coef 0.5 --num-processes 16 --num-steps 65 --num-mini-batch 4 --log-interval 1 --use-linear-lr-decay --entropy-coef 0.02 --lrCL 20e-4 --choiceCLparams 0 --seed 80000 --head 8 --factorR 0.5 -``` diff --git a/3DOrangeTree/Utils/__pycache__/utils.cpython-36.pyc b/3DOrangeTree/Utils/__pycache__/utils.cpython-36.pyc deleted file mode 100644 index 4a59eac..0000000 Binary files a/3DOrangeTree/Utils/__pycache__/utils.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/Utils/__pycache__/utils.cpython-37.pyc b/3DOrangeTree/Utils/__pycache__/utils.cpython-37.pyc deleted file mode 100644 index d40d20f..0000000 Binary files a/3DOrangeTree/Utils/__pycache__/utils.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/Utils/__pycache__/utils.cpython-38.pyc b/3DOrangeTree/Utils/__pycache__/utils.cpython-38.pyc deleted file mode 100644 index 7864af5..0000000 Binary files a/3DOrangeTree/Utils/__pycache__/utils.cpython-38.pyc and /dev/null differ diff --git a/3DOrangeTree/Utils/utils.py b/3DOrangeTree/Utils/utils.py deleted file mode 100644 index 2f21b7e..0000000 --- a/3DOrangeTree/Utils/utils.py +++ /dev/null @@ -1,10 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt - -def plot_learning_curve(x, scores, figure_file): - running_avg = np.zeros(len(scores)) - for i in range(len(running_avg)): - running_avg[i] = np.mean(scores[max(0, i-10):(i+1)]) - plt.plot(x, running_avg) - plt.title('Running average of previous 10 scores') - plt.savefig(figure_file) \ No newline at end of file diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index b0a25ee..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/arguments.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/arguments.cpython-37.pyc deleted file mode 100644 index 7a9cf76..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/arguments.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/argumentstvt.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/argumentstvt.cpython-37.pyc deleted file mode 100644 index e44f6fd..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/argumentstvt.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/distributions.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/distributions.cpython-37.pyc deleted file mode 100644 index 33f073e..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/distributions.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/envs.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/envs.cpython-37.pyc deleted file mode 100644 index c5d5bb4..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/envs.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/model.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/model.cpython-37.pyc deleted file mode 100644 index f4fc315..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/model.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/model384.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/model384.cpython-37.pyc deleted file mode 100644 index 3b0fbca..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/model384.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCL.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCL.cpython-37.pyc deleted file mode 100644 index d61d219..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCL.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCL64.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCL64.cpython-37.pyc deleted file mode 100644 index a86a6c7..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCL64.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCLrnn.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCLrnn.cpython-37.pyc deleted file mode 100644 index df54754..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCLrnn.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCLrnn54.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCLrnn54.cpython-37.pyc deleted file mode 100644 index dce2905..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelCLrnn54.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelConspec.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelConspec.cpython-37.pyc deleted file mode 100644 index 3ea5480..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelConspec.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelRL.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelRL.cpython-37.pyc deleted file mode 100644 index 8673731..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelRL.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpala.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpala.cpython-37.pyc deleted file mode 100644 index 9c7044b..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpala.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpala64.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpala64.cpython-37.pyc deleted file mode 100644 index a3a2b6a..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpala64.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpalarnn.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpalarnn.cpython-37.pyc deleted file mode 100644 index d0b1cf4..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpalarnn.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpalarnn54.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpalarnn54.cpython-37.pyc deleted file mode 100644 index 896c1b8..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/modelsepmini384dkdecodeImpalarnn54.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storage.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/storage.cpython-37.pyc deleted file mode 100644 index 9f44fd7..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storage.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY.cpython-37.pyc deleted file mode 100644 index 56676fe..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2.cpython-37.pyc deleted file mode 100644 index 96cdc91..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2x.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2x.cpython-37.pyc deleted file mode 100644 index 90ed8d6..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2x.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2x22.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2x22.cpython-37.pyc deleted file mode 100644 index 647e5ef..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2x22.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2xx.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2xx.cpython-37.pyc deleted file mode 100644 index d1bd36e..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCL84UNITY2xx.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCLminiTVTnobetter.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCLminiTVTnobetter.cpython-37.pyc deleted file mode 100644 index a9906a7..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/storageCLminiTVTnobetter.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/__pycache__/utils.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/__pycache__/utils.cpython-37.pyc deleted file mode 100644 index faa1ed0..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/__pycache__/utils.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/__init__.py b/3DOrangeTree/a2c_ppo_acktr/algo/__init__.py deleted file mode 100644 index 05df2e4..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/algo/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ - - -from .ppoConspec import PPOConspec -from .ppoConspectest import PPOConspectest - - diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index ce26f78..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/gail.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/gail.cpython-37.pyc deleted file mode 100644 index 84f72ec..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/gail.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/ppoConspec.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/ppoConspec.cpython-37.pyc deleted file mode 100644 index 727cae0..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/ppoConspec.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/ppoConspectest.cpython-37.pyc b/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/ppoConspectest.cpython-37.pyc deleted file mode 100644 index dd785be..0000000 Binary files a/3DOrangeTree/a2c_ppo_acktr/algo/__pycache__/ppoConspectest.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/ppoConspec.py b/3DOrangeTree/a2c_ppo_acktr/algo/ppoConspec.py deleted file mode 100644 index c1fe9fc..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/algo/ppoConspec.py +++ /dev/null @@ -1,196 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - - -class PPOConspec(): - def __init__(self, - actor_critic,actor_criticCL, - clip_param, - ppo_epoch, - num_mini_batch, - value_loss_coef, - entropy_coef, - module, - choiceCLparams, - args, - lrCL=None, - lr=None, - eps=None, - max_grad_norm=None, - use_clipped_value_loss=True): - - self.actor_critic = actor_critic - self.actor_criticCL = actor_criticCL - - self.clip_param = clip_param - self.ppo_epoch = ppo_epoch - self.num_mini_batch = num_mini_batch - - self.value_loss_coef = value_loss_coef - self.entropy_coef = entropy_coef - - self.max_grad_norm = max_grad_norm - self.use_clipped_value_loss = use_clipped_value_loss - self.moduleCL = module - self.args = args - self.listparams = list(actor_criticCL.parameters()) + list(self.moduleCL.parameters()) - self.optimizer = optim.Adam(actor_critic.parameters(), lr=lr, eps=eps) - self.optimizerCL = optim.Adam(self.listparams, lr=lrCL, eps=eps) - - def fictitiousReward(self,rollouts,keysUsed,device,iteration): - obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = rollouts.feed_attnR() - _, _, _, _, hidden = self.actor_criticCL.evaluate_actionsHiddens( - obs_batch, recurrent_hidden_states_batch, masks_batch, - actions_batch) - hidden = hidden.view(*obs_batchorig.size()[:2],-1) - _, attentionCLattn, _ = self.moduleCL(hidden, reward_batch, -1, obs_batchorig, 111, self.args.seed,0) - attentionCLattnnp = attentionCLattn.view(*obs_batchorig.size()[:2],-1) - goodz,_ = torch.max(attentionCLattnnp, dim=0) - size1, size2, size3 = attentionCLattnnp.size() - greater = ((attentionCLattnnp > 0.6)) * attentionCLattnnp - filterCorr = torch.tile(torch.reshape(keysUsed, (1, 1, -1)), (size1, size2, 1)) - filterCorr1 = filterCorr * self.args.factorR - sendContrastvalue = (greater * filterCorr1) - roundhalf = 3 - round = roundhalf - allvalues = [] - for orthoit in range(round * 2 + 1): - temp = torch.roll(sendContrastvalue, orthoit - roundhalf, dims=0) - if orthoit - roundhalf > 0: - temp[:(orthoit - roundhalf)] = 0. - allvalues.append(temp) - allvalues = torch.stack(allvalues, dim=0) - allvaluesmax, _ = torch.max(allvalues, dim=0) - - allvaluesdifference = sendContrastvalue - allvaluesmax - sendContrastvalue[allvaluesdifference < 0.] = 0. - sendContrastvalue[sendContrastvalue < 0.] = 0. - sendContrastvaluesummed = 0. - for orthoit in range(round): - temp = torch.roll(sendContrastvalue, orthoit + 1, dims=0) * (.5 ** (orthoit)) - temp[:orthoit] = 0. - sendContrastvaluesummed += temp - sendContrastvaluesummed = sendContrastvaluesummed - sendContrastvalue = sendContrastvalue - sendContrastvaluesummed - sendContrastvalue[sendContrastvalue < 0.] = 0. - sendContrastvalue = sendContrastvalue.sum(2) - rollouts.contrastvalueReward(sendContrastvalue) - - def update(self, rollouts, head, keysUsed,goodones,iterate): - advantages = rollouts.returns[:-1] - rollouts.value_preds[:-1] - advantages = (advantages - advantages.mean()) / ( - advantages.std() + 1e-5) - - value_loss_epoch = 0 - action_loss_epoch = 0 - dist_entropy_epoch = 0 - if self.args.factorR < 0.01: - pass - else: - if rollouts.stepS > rollouts.success -1: - wwtotalpos = [] - wwtotalneg = [] - attentionCL = [] - costCL = 0 - for iii in range(head): - if keysUsed[iii] > 0.5: - obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = rollouts.feed_attnRSFheads(iii) - _, _, _, _, hidden = self.actor_criticCL.evaluate_actionsHiddens( - obs_batch, recurrent_hidden_states_batch, masks_batch, - actions_batch) - hidden = hidden.view(*obs_batchorig.size()[:2], -1) - costCL0, attentionCL0, ww = self.moduleCL(hidden, reward_batch, iii, obs_batchorig, iterate, self.args.seed, 1) - attentionCL.append(attentionCL0[:, :, iii].squeeze().transpose(1, 0)) - costCL += costCL0 - wwtotalpos.append(ww[0][iii].detach().cpu()) - wwtotalneg.append(ww[1][iii].detach().cpu()) - else: - obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = rollouts.feed_attnRSF() - - _, _, _, _, hidden = self.actor_criticCL.evaluate_actionsHiddens( - obs_batch, recurrent_hidden_states_batch, masks_batch, - actions_batch) - hidden = hidden.view(*obs_batchorig.size()[:2], -1) - costCL0, attentionCL0, ww = self.moduleCL(hidden, reward_batch, iii, obs_batchorig, iterate, self.args.seed,0) - cossimtotalmaxxx, _ = (torch.max(attentionCL0[:,:,iii], dim=0)) - attentionCL.append(attentionCL0[:, :, iii].squeeze().transpose(1, 0)) - costCL += costCL0 - wwtotalpos.append(ww[0][iii].detach().cpu()) - wwtotalneg.append(ww[1][iii].detach().cpu()) - - for indall in range(head): - if (wwtotalpos[indall] - wwtotalneg[indall] > 0.6) and wwtotalpos[indall] > 0.6: - goodones[indall] += 1 - else: - goodones[indall] = 0 - for iii in range(head): - if goodones[iii] > 25 and keysUsed[iii] < 0.1: - keysUsed[iii] = 1. - rollouts.storeheadsSF(iii) - - - self.optimizerCL.zero_grad() - costCL.backward() - self.optimizerCL.step() - - - for e in range(self.ppo_epoch): - if self.actor_critic.is_recurrent: - data_generator = rollouts.recurrent_generator( - advantages, self.num_mini_batch) - else: - data_generator = rollouts.feed_forward_generator( - advantages, self.num_mini_batch) - - for sample in data_generator: - obs_batch, recurrent_hidden_states_batch, actions_batch, \ - value_preds_batch, return_batch, masks_batch, old_action_log_probs_batch, \ - adv_targ = sample - - # Reshape to do in a single forward pass for all steps - values, action_log_probs, dist_entropy, _ = self.actor_critic.evaluate_actions( - obs_batch, recurrent_hidden_states_batch, masks_batch, - actions_batch) - - - - ratio = torch.exp(action_log_probs - - old_action_log_probs_batch) - surr1 = ratio * adv_targ - surr2 = torch.clamp(ratio, 1.0 - self.clip_param, - 1.0 + self.clip_param) * adv_targ - action_loss = -torch.min(surr1, surr2).mean() - - if self.use_clipped_value_loss: - value_pred_clipped = value_preds_batch + \ - (values - value_preds_batch).clamp(-self.clip_param, self.clip_param) - value_losses = (values - return_batch).pow(2) - value_losses_clipped = ( - value_pred_clipped - return_batch).pow(2) - value_loss = 0.5 * torch.max(value_losses, - value_losses_clipped).mean() - else: - value_loss = 0.5 * (return_batch - values).pow(2).mean() - #''' - self.optimizer.zero_grad() - (value_loss * self.value_loss_coef + action_loss - - dist_entropy * self.entropy_coef).backward() - nn.utils.clip_grad_norm_(self.actor_critic.parameters(), - self.max_grad_norm) - - self.optimizer.step() - #''' - - value_loss_epoch += value_loss.item() - action_loss_epoch += action_loss.item() - dist_entropy_epoch += dist_entropy.item() - - num_updates = self.ppo_epoch * self.num_mini_batch - - value_loss_epoch /= num_updates - action_loss_epoch /= num_updates - dist_entropy_epoch /= num_updates - - return value_loss_epoch, action_loss_epoch, dist_entropy_epoch, keysUsed, goodones diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/ppoConspectest.py b/3DOrangeTree/a2c_ppo_acktr/algo/ppoConspectest.py deleted file mode 100644 index 396ba8f..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/algo/ppoConspectest.py +++ /dev/null @@ -1,147 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - - -class PPOConspectest(): - def __init__(self, - actor_critic,actor_criticCL, - clip_param, - ppo_epoch, - num_mini_batch, - value_loss_coef, - entropy_coef, - module, - choiceCLparams, - args, - lrCL=None, - lr=None, - eps=None, - max_grad_norm=None, - use_clipped_value_loss=True): - - self.actor_critic = actor_critic - self.actor_criticCL = actor_criticCL - - self.clip_param = clip_param - self.ppo_epoch = ppo_epoch - self.num_mini_batch = num_mini_batch - - self.value_loss_coef = value_loss_coef - self.entropy_coef = entropy_coef - - self.max_grad_norm = max_grad_norm - self.use_clipped_value_loss = use_clipped_value_loss - self.moduleCL = module - self.args = args - self.listparams = list(actor_criticCL.parameters()) + list(self.moduleCL.parameters()) - self.optimizer = optim.Adam(actor_critic.parameters(), lr=lr, eps=eps) - self.optimizerCL = optim.Adam(self.listparams, lr=lrCL, eps=eps) - - def fictitiousReward(self,rollouts,keysUsed,device,iteration): - obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = rollouts.feed_attnR() - _, _, _, _, hidden = self.actor_criticCL.evaluate_actionsHiddens( - obs_batch, recurrent_hidden_states_batch, masks_batch, - actions_batch) - - hidden = hidden.view(*obs_batchorig.size()[:2],-1) - _, attentionCLattn, _ = self.moduleCL(hidden, reward_batch, -1, obs_batchorig, 111, self.args.seed,0) - - attentionCLattnnp = attentionCLattn.view(*obs_batchorig.size()[:2],-1) - goodz,_ = torch.max(attentionCLattnnp, dim=0) - size1, size2, size3 = attentionCLattnnp.size() - - greater = ((attentionCLattnnp > 0.6)) * attentionCLattnnp #### - filterCorr = torch.tile(torch.reshape(keysUsed, (1, 1, -1)), (size1, size2, 1)) - filterCorr1 = filterCorr * self.args.factorR - sendContrastvalue = (greater * filterCorr1) - - roundhalf = 3 - round = roundhalf - allvalues = [] - for orthoit in range(round * 2 + 1): - temp = torch.roll(sendContrastvalue, orthoit - roundhalf, dims=0) - if orthoit - roundhalf > 0: - temp[:(orthoit - roundhalf)] = 0. - allvalues.append(temp) - allvalues = torch.stack(allvalues, dim=0) - allvaluesmax, _ = torch.max(allvalues, dim=0) - - allvaluesdifference = sendContrastvalue - allvaluesmax - sendContrastvalue[allvaluesdifference < 0.] = 0. - sendContrastvalue[sendContrastvalue < 0.] = 0. - - sendContrastvaluesummed = 0. - for orthoit in range(round): - temp = torch.roll(sendContrastvalue, orthoit + 1, dims=0) * (.5 ** (orthoit)) - temp[:orthoit] = 0. - sendContrastvaluesummed += temp - sendContrastvaluesummed = sendContrastvaluesummed - sendContrastvalue = sendContrastvalue - sendContrastvaluesummed - sendContrastvalue[sendContrastvalue < 0.] = 0. - sendContrastvalue = sendContrastvalue.sum(2) - rollouts.contrastvalueReward(sendContrastvalue) - def update(self, rollouts, head, keysUsed,goodones,iterate): - advantages = rollouts.returns[:-1] - rollouts.value_preds[:-1] - advantages = (advantages - advantages.mean()) / ( - advantages.std() + 1e-5) - - value_loss_epoch = 0 - action_loss_epoch = 0 - dist_entropy_epoch = 0 - for e in range(self.ppo_epoch): - if self.actor_critic.is_recurrent: - data_generator = rollouts.recurrent_generator( - advantages, self.num_mini_batch) - else: - data_generator = rollouts.feed_forward_generator( - advantages, self.num_mini_batch) - - for sample in data_generator: - obs_batch, recurrent_hidden_states_batch, actions_batch, \ - value_preds_batch, return_batch, masks_batch, old_action_log_probs_batch, \ - adv_targ = sample - - # Reshape to do in a single forward pass for all steps - values, action_log_probs, dist_entropy, _ = self.actor_critic.evaluate_actions( - obs_batch, recurrent_hidden_states_batch, masks_batch, - actions_batch) - - - ratio = torch.exp(action_log_probs - - old_action_log_probs_batch) - surr1 = ratio * adv_targ - surr2 = torch.clamp(ratio, 1.0 - self.clip_param, - 1.0 + self.clip_param) * adv_targ - action_loss = -torch.min(surr1, surr2).mean() - - if self.use_clipped_value_loss: - value_pred_clipped = value_preds_batch + \ - (values - value_preds_batch).clamp(-self.clip_param, self.clip_param) - value_losses = (values - return_batch).pow(2) - value_losses_clipped = ( - value_pred_clipped - return_batch).pow(2) - value_loss = 0.5 * torch.max(value_losses, - value_losses_clipped).mean() - else: - value_loss = 0.5 * (return_batch - values).pow(2).mean() - self.optimizer.zero_grad() - (value_loss * self.value_loss_coef + action_loss - - dist_entropy * self.entropy_coef).backward() - nn.utils.clip_grad_norm_(self.actor_critic.parameters(), - self.max_grad_norm) - - self.optimizer.step() - - value_loss_epoch += value_loss.item() - action_loss_epoch += action_loss.item() - dist_entropy_epoch += dist_entropy.item() - - num_updates = self.ppo_epoch * self.num_mini_batch - - value_loss_epoch /= num_updates - action_loss_epoch /= num_updates - dist_entropy_epoch /= num_updates - - return value_loss_epoch, action_loss_epoch, dist_entropy_epoch, keysUsed, goodones diff --git a/3DOrangeTree/a2c_ppo_acktr/arguments.py b/3DOrangeTree/a2c_ppo_acktr/arguments.py deleted file mode 100644 index 25ac521..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/arguments.py +++ /dev/null @@ -1,190 +0,0 @@ -import argparse - -import torch - - -def get_args(): - parser = argparse.ArgumentParser(description='RL') - parser.add_argument( - '--head', - type=int, - default=3, - help='gail batch size (default: 128)') - parser.add_argument( - '--choiceCLparams', - type=int, - default=0, - help='gail batch size (default: 128)') - parser.add_argument( - '--skip', - type=int, - default=4, - help='gail batch size (default: 128)') - parser.add_argument( - '--lrCL', type=float, default=7e-4, help='learning rate (default: 7e-4)') - parser.add_argument( - '--algo', default='ppoCL', help='algorithm to use: a2c | ppo | acktr') - parser.add_argument( - '--gail', - action='store_true', - default=False, - help='do imitation learning with gail') - parser.add_argument( - '--gail-experts-dir', - default='./gail_experts', - help='directory that contains expert demonstrations for gail') - parser.add_argument( - '--gail-batch-size', - type=int, - default=128, - help='gail batch size (default: 128)') - parser.add_argument( - '--gail-epoch', type=int, default=5, help='gail epochs (default: 5)') - parser.add_argument( - '--lr', type=float, default=7e-4, help='learning rate (default: 7e-4)') - parser.add_argument( - '--factorR', type=float, default=10., help='learning rate (default: 7e-4)') - parser.add_argument( - '--factorC', type=float, default=10., help='learning rate (default: 7e-4)') - parser.add_argument( - '--stability', type=float, default=1., help='learning rate (default: 7e-4)') - parser.add_argument( - '--eps', - type=float, - default=1e-5, - help='RMSprop optimizer epsilon (default: 1e-5)') - parser.add_argument( - '--alpha', - type=float, - default=0.99, - help='RMSprop optimizer apha (default: 0.99)') - parser.add_argument( - '--gamma', - type=float, - default=0.99, - help='discount factor for rewards (default: 0.99)') - parser.add_argument( - '--use-gae', - action='store_true', - default=False, - help='use generalized advantage estimation') - parser.add_argument( - '--gae-lambda', - type=float, - default=0.95, - help='gae lambda parameter (default: 0.95)') - parser.add_argument( - '--entropy-coef', - type=float, - default=0.01, - help='entropy term coefficient (default: 0.01)') - parser.add_argument( - '--value-loss-coef', - type=float, - default=0.5, - help='value loss coefficient (default: 0.5)') - parser.add_argument( - '--max-grad-norm', - type=float, - default=0.5, - help='max norm of gradients (default: 0.5)') - parser.add_argument( - '--seed', type=int, default=1, help='random seed (default: 1)') - parser.add_argument( - '--cuda-deterministic', - action='store_true', - default=False, - help="sets flags for determinism when using CUDA (potentially slow!)") - parser.add_argument( - '--num-processes', - type=int, - default=16, - help='how many training CPU processes to use (default: 16)') - parser.add_argument( - '--num-steps', - type=int, - default=5, - help='number of forward steps in A2C (default: 5)') - parser.add_argument( - '--ppo-epoch', - type=int, - default=4, - help='number of ppo epochs (default: 4)') - parser.add_argument( - '--num-mini-batch', - type=int, - default=32, - help='number of batches for ppo (default: 32)') - parser.add_argument( - '--clip-param', - type=float, - default=0.2, - help='ppo clip parameter (default: 0.2)') - parser.add_argument( - '--log-interval', - type=int, - default=10, - help='log interval, one log per n updates (default: 10)') - parser.add_argument( - '--save-interval', - type=int, - default=100, - help='save interval, one save per n updates (default: 100)') - parser.add_argument( - '--eval-interval', - type=int, - default=None, - help='eval interval, one eval per n updates (default: None)') - parser.add_argument( - '--num-env-steps', - type=int, - default=10e8, - help='number of environment steps to train (default: 10e6)') - parser.add_argument( - '--env-name', - default='PongNoFrameskip-v4', - help='environment to train on (default: PongNoFrameskip-v4)') - parser.add_argument( - '--log-dir', - default='/tmp/gym/', - help='directory to save agent logs (default: /tmp/gym)') - parser.add_argument( - '--save-dir', - default='./trained_models/', - help='directory to save agent logs (default: ./trained_models/)') - parser.add_argument( - '--no-cuda', - action='store_true', - default=False, - help='disables CUDA training') - parser.add_argument( - '--use-proper-time-limits', - action='store_true', - default=False, - help='compute returns taking into account time limits') - parser.add_argument( - '--recurrent-policy', - action='store_false', - default=True, - help='use a recurrent policy') - #parser.add_argument( - # '--recurrent-policy', - # action='store_true', - # default=False, - # help='use a recurrent policy') - parser.add_argument( - '--use-linear-lr-decay', - action='store_true', - default=False, - help='use a linear schedule on the learning rate') - args = parser.parse_args() - - args.cuda = not args.no_cuda and torch.cuda.is_available() - ################I REMOVED THIS - #assert args.algo in ['a2c', 'ppo', 'acktr'] - #if args.recurrent_policy: - # assert args.algo in ['a2c', 'ppo'], \ - # 'Recurrent policy is not implemented for ACKTR' - ################I REMOVED THIS - - return args diff --git a/3DOrangeTree/a2c_ppo_acktr/envs.py b/3DOrangeTree/a2c_ppo_acktr/envs.py deleted file mode 100644 index 2d9f316..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/envs.py +++ /dev/null @@ -1,259 +0,0 @@ -import os - -import gym -import numpy as np -import torch -from gym.spaces.box import Box -from gym.wrappers.clip_action import ClipAction -from stable_baselines3.common.atari_wrappers import (ClipRewardEnv, - EpisodicLifeEnv, - FireResetEnv, - MaxAndSkipEnv, - NoopResetEnv, WarpFrame) -from stable_baselines3.common.monitor import Monitor -from stable_baselines3.common.vec_env import (DummyVecEnv, SubprocVecEnv, - VecEnvWrapper) -from stable_baselines3.common.vec_env.vec_normalize import \ - VecNormalize as VecNormalize_ - -try: - import dmc2gym -except ImportError: - pass - -try: - import roboschool -except ImportError: - pass - -try: - import pybullet_envs -except ImportError: - pass - - -def make_env(env_id, seed, rank, log_dir, allow_early_resets): - def _thunk(): - if env_id.startswith("dm"): - _, domain, task = env_id.split('.') - env = dmc2gym.make(domain_name=domain, task_name=task) - env = ClipAction(env) - else: - env = gym.make(env_id) - - is_atari = hasattr(gym.envs, 'atari') and isinstance( - env.unwrapped, gym.envs.atari.atari_env.AtariEnv) - if is_atari: - env = NoopResetEnv(env, noop_max=30) - env = MaxAndSkipEnv(env, skip=4) - - env.seed(seed + rank) - - if str(env.__class__.__name__).find('TimeLimit') >= 0: - env = TimeLimitMask(env) - - #if log_dir is not None: - env = Monitor(env, - os.path.join(log_dir, str(rank)), - allow_early_resets=allow_early_resets) - - if is_atari: - if len(env.observation_space.shape) == 3: - env = EpisodicLifeEnv(env) - if "FIRE" in env.unwrapped.get_action_meanings(): - env = FireResetEnv(env) - env = WarpFrame(env, width=84, height=84) - env = ClipRewardEnv(env) - elif len(env.observation_space.shape) == 3: - raise NotImplementedError( - "CNN models work only for atari,\n" - "please use a custom wrapper for a custom pixel input env.\n" - "See wrap_deepmind for an example.") - - # If the input has shape (W,H,3), wrap for PyTorch convolutions - obs_shape = env.observation_space.shape - if len(obs_shape) == 3 and obs_shape[2] in [1, 3]: - env = TransposeImage(env, op=[2, 0, 1]) - - return env - - return _thunk - - -def make_vec_envs(env_name, - seed, - num_processes, - gamma, - log_dir, - device, - allow_early_resets, - num_frame_stack=None): - envs = [ - make_env(env_name, seed, i, log_dir, allow_early_resets) - for i in range(num_processes) - ] - - if len(envs) > 1: - envs = SubprocVecEnv(envs) - else: - envs = DummyVecEnv(envs) - - if len(envs.observation_space.shape) == 1: - if gamma is None: - envs = VecNormalize(envs, norm_reward=False) - else: - envs = VecNormalize(envs, gamma=gamma) - - envs = VecPyTorch(envs, device) - - if num_frame_stack is not None: - envs = VecPyTorchFrameStack(envs, num_frame_stack, device) - elif len(envs.observation_space.shape) == 3: - envs = VecPyTorchFrameStack(envs, 4, device) - - return envs - - -# Checks whether done was caused my timit limits or not -class TimeLimitMask(gym.Wrapper): - def step(self, action): - obs, rew, done, info = self.env.step(action) - if done and self.env._max_episode_steps == self.env._elapsed_steps: - info['bad_transition'] = True - - return obs, rew, done, info - - def reset(self, **kwargs): - return self.env.reset(**kwargs) - - -# Can be used to test recurrent policies for Reacher-v2 -class MaskGoal(gym.ObservationWrapper): - def observation(self, observation): - if self.env._elapsed_steps > 0: - observation[-2:] = 0 - return observation - - -class TransposeObs(gym.ObservationWrapper): - def __init__(self, env=None): - """ - Transpose observation space (base class) - """ - super(TransposeObs, self).__init__(env) - - -class TransposeImage(TransposeObs): - def __init__(self, env=None, op=[2, 0, 1]): - """ - Transpose observation space for images - """ - super(TransposeImage, self).__init__(env) - assert len(op) == 3, "Error: Operation, " + str(op) + ", must be dim3" - self.op = op - obs_shape = self.observation_space.shape - self.observation_space = Box( - self.observation_space.low[0, 0, 0], - self.observation_space.high[0, 0, 0], [ - obs_shape[self.op[0]], obs_shape[self.op[1]], - obs_shape[self.op[2]] - ], - dtype=self.observation_space.dtype) - - def observation(self, ob): - return ob.transpose(self.op[0], self.op[1], self.op[2]) - - -class VecPyTorch(VecEnvWrapper): - def __init__(self, venv, device): - """Return only every `skip`-th frame""" - super(VecPyTorch, self).__init__(venv) - self.device = device - # TODO: Fix data types - - def reset(self): - obs = self.venv.reset() - obs = torch.from_numpy(obs).float().to(self.device) - return obs - - def step_async(self, actions): - if isinstance(actions, torch.LongTensor): - # Squeeze the dimension for discrete actions - actions = actions.squeeze(1) - actions = actions.cpu().numpy() - self.venv.step_async(actions) - - def step_wait(self): - obs, reward, done, info = self.venv.step_wait() - obs = torch.from_numpy(obs).float().to(self.device) - reward = torch.from_numpy(reward).unsqueeze(dim=1).float() - return obs, reward, done, info - - -class VecNormalize(VecNormalize_): - def __init__(self, *args, **kwargs): - super(VecNormalize, self).__init__(*args, **kwargs) - self.training = True - - def _obfilt(self, obs, update=True): - if self.obs_rms: - if self.training and update: - self.obs_rms.update(obs) - obs = np.clip((obs - self.obs_rms.mean) / - np.sqrt(self.obs_rms.var + self.epsilon), - -self.clip_obs, self.clip_obs) - return obs - else: - return obs - - def train(self): - self.training = True - - def eval(self): - self.training = False - - -# Derived from -# https://github.com/openai/baselines/blob/master/baselines/common/vec_env/vec_frame_stack.py -class VecPyTorchFrameStack(VecEnvWrapper): - def __init__(self, venv, nstack, device=None): - self.venv = venv - self.nstack = nstack - - wos = venv.observation_space # wrapped ob space - self.shape_dim0 = wos.shape[0] - - low = np.repeat(wos.low, self.nstack, axis=0) - high = np.repeat(wos.high, self.nstack, axis=0) - - if device is None: - device = torch.device('cpu') - self.stacked_obs = torch.zeros((venv.num_envs, ) + - low.shape).to(device) - - observation_space = gym.spaces.Box(low=low, - high=high, - dtype=venv.observation_space.dtype) - VecEnvWrapper.__init__(self, venv, observation_space=observation_space) - - def step_wait(self): - obs, rews, news, infos = self.venv.step_wait() - self.stacked_obs[:, :-self.shape_dim0] = \ - self.stacked_obs[:, self.shape_dim0:].clone() - for (i, new) in enumerate(news): - if new: - self.stacked_obs[i] = 0 - self.stacked_obs[:, -self.shape_dim0:] = obs - return self.stacked_obs, rews, news, infos - - def reset(self): - obs = self.venv.reset() - if torch.backends.cudnn.deterministic: - self.stacked_obs = torch.zeros(self.stacked_obs.shape) - else: - self.stacked_obs.zero_() - self.stacked_obs[:, -self.shape_dim0:] = obs - return self.stacked_obs - - def close(self): - self.venv.close() diff --git a/3DOrangeTree/a2c_ppo_acktr/modelConspec.py b/3DOrangeTree/a2c_ppo_acktr/modelConspec.py deleted file mode 100644 index 329ba9e..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/modelConspec.py +++ /dev/null @@ -1,164 +0,0 @@ -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - -from a2c_ppo_acktr.distributions import Bernoulli, Categorical, DiagGaussian -from a2c_ppo_acktr.utils import init -from misc_util import orthogonal_init, xavier_uniform_init - - - -class Flatten(nn.Module): - def forward(self, x): - return x.view(x.size(0), -1) - - -class PolicyCL(nn.Module): - def __init__(self, obs_shape, action_space, base=None, base_kwargs=None): - super(PolicyCL, self).__init__() - if base_kwargs is None: - base_kwargs = {} - if base is None: - if len(obs_shape) == 3: - base = ImpalaModel - - else: - raise NotImplementedError - - self.base = base(obs_shape[0], **base_kwargs) - - if action_space.__class__.__name__ == "Discrete": - num_outputs = action_space.n - self.dist = Categorical(self.base.output_size, num_outputs) - elif action_space.__class__.__name__ == "Box": - num_outputs = action_space.shape[0] - self.dist = DiagGaussian(self.base.output_size, num_outputs) - elif action_space.__class__.__name__ == "MultiBinary": - num_outputs = action_space.shape[0] - self.dist = Bernoulli(self.base.output_size, num_outputs) - else: - raise NotImplementedError - - @property - def is_recurrent(self): - return self.base.is_recurrent - - @property - def recurrent_hidden_state_size(self): - """Size of rnn_hx.""" - return self.base.recurrent_hidden_state_size - - def forward(self, inputs, rnn_hxs, masks): - raise NotImplementedError - - def act(self, inputs, rnn_hxs, masks, deterministic=False): - value, actor_features, rnn_hxs = self.base(inputs, rnn_hxs, masks) - dist = self.dist(actor_features) - - if deterministic: - action = dist.mode() - else: - action = dist.sample() - - action_log_probs = dist.log_probs(action) - dist_entropy = dist.entropy().mean() - - return value, action, action_log_probs, rnn_hxs - - def get_value(self, inputs, rnn_hxs, masks): - value, _, _ = self.base(inputs, rnn_hxs, masks) - return value - - def evaluate_actions(self, inputs, rnn_hxs, masks, action): - value, actor_features, rnn_hxs = self.base(inputs, rnn_hxs, masks) - dist = self.dist(actor_features) - - action_log_probs = dist.log_probs(action) - dist_entropy = dist.entropy().mean() - - return value, action_log_probs, dist_entropy, rnn_hxs - def evaluate_actionsHiddens(self, inputs, rnn_hxs, masks, action): - value, actor_features, rnn_hxs = self.base(inputs, rnn_hxs, masks) - dist = self.dist(actor_features) - - action_log_probs = dist.log_probs(action) - dist_entropy = dist.entropy().mean() - - return value, action_log_probs, dist_entropy, rnn_hxs,actor_features - - - -class ResidualBlock(nn.Module): - def __init__(self, - in_channels): - super(ResidualBlock, self).__init__() - self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=1, padding=1) - self.conv2 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=1, padding=1) - - def forward(self, x): - out = nn.ReLU()(x) - out = self.conv1(out) - out = nn.ReLU()(out) - out = self.conv2(out) - #print(wefw) - return out + x - -class ImpalaBlock(nn.Module): - def __init__(self, in_channels, out_channels): - super(ImpalaBlock, self).__init__() - self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1, padding=1) - self.res1 = ResidualBlock(out_channels) - self.res2 = ResidualBlock(out_channels) - - def forward(self, x): - x = self.conv(x) - x = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)(x) - x = self.res1(x) - x = self.res2(x) - return x - -class ImpalaModel(nn.Module): - def __init__(self,in_channels, recurrent=False, hidden_size=256): - super(ImpalaModel, self).__init__() - self.block1 = ImpalaBlock(in_channels=in_channels, out_channels=8) - self.block2 = ImpalaBlock(in_channels=8, out_channels=16) - self.block3 = ImpalaBlock(in_channels=16, out_channels=16) - self.fc = nn.Linear(in_features=1936, out_features=hidden_size) - - self.output_dim = 256 - self.apply(xavier_uniform_init) - init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init. - constant_(x, 0), nn.init.calculate_gain('relu')) - self.critic_linear = init_(nn.Linear(hidden_size, 1)) - self.train() - - self._recurrent = recurrent - self._hidden_size = hidden_size - - @property - def is_recurrent(self): - return self._recurrent - - @property - def recurrent_hidden_state_size(self): - if self._recurrent: - return self._hidden_size - return 1 - - @property - def output_size(self): - return self._hidden_size - - - def forward(self, x, rnn_hxs, masks): - x = self.block1( x/ 255.0) - x = self.block2(x) - x = self.block3(x) - x = nn.ReLU()(x) - x = Flatten()(x) - x = self.fc(x) - x = nn.ReLU()(x) - - return self.critic_linear(x), x, rnn_hxs - diff --git a/3DOrangeTree/a2c_ppo_acktr/modelRL.py b/3DOrangeTree/a2c_ppo_acktr/modelRL.py deleted file mode 100644 index e638ac4..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/modelRL.py +++ /dev/null @@ -1,157 +0,0 @@ -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - -from a2c_ppo_acktr.distributions import Bernoulli, Categorical, DiagGaussian -from a2c_ppo_acktr.utils import init -from misc_util import orthogonal_init, xavier_uniform_init - - -class Flatten(nn.Module): - def forward(self, x): - return x.view(x.size(0), -1) - - -class Policy(nn.Module): - def __init__(self, obs_shape, action_space, base=None, base_kwargs=None): - super(Policy, self).__init__() - if base_kwargs is None: - base_kwargs = {} - if base is None: - if len(obs_shape) == 3: - base = ImpalaModel - - else: - raise NotImplementedError - - self.base = base(obs_shape[0], **base_kwargs) - - - self.dist = Categorical(self.base.output_size, 4) - - - @property - def is_recurrent(self): - return self.base.is_recurrent - - @property - def recurrent_hidden_state_size(self): - """Size of rnn_hx.""" - return self.base.recurrent_hidden_state_size - - def forward(self, inputs, rnn_hxs, masks): - raise NotImplementedError - - def act(self, inputs, rnn_hxs, masks, deterministic=False): - value, actor_features, rnn_hxs = self.base(inputs, rnn_hxs, masks) - dist = self.dist(actor_features) - - if deterministic: - action = dist.mode() - else: - action = dist.sample() - - action_log_probs = dist.log_probs(action) - dist_entropy = dist.entropy().mean() - - return value, action, action_log_probs, rnn_hxs - - def get_value(self, inputs, rnn_hxs, masks): - value, _, _ = self.base(inputs, rnn_hxs, masks) - return value - - - def evaluate_actions(self, inputs, rnn_hxs, masks, action): - value, actor_features, rnn_hxs = self.base(inputs, rnn_hxs, masks) - dist = self.dist(actor_features) - - action_log_probs = dist.log_probs(action) - dist_entropy = dist.entropy().mean() - - return value, action_log_probs, dist_entropy, rnn_hxs - def evaluate_actionsHiddens(self, inputs, rnn_hxs, masks, action): - value, actor_features, rnn_hxs = self.base(inputs, rnn_hxs, masks) - dist = self.dist(actor_features) - - action_log_probs = dist.log_probs(action) - dist_entropy = dist.entropy().mean() - - return value, action_log_probs, dist_entropy, rnn_hxs,actor_features - - - -class ResidualBlock(nn.Module): - def __init__(self, - in_channels): - super(ResidualBlock, self).__init__() - self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=1, padding=1) - self.conv2 = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=1, padding=1) - - def forward(self, x): - out = nn.ReLU()(x) - out = self.conv1(out) - out = nn.ReLU()(out) - out = self.conv2(out) - #print(wefw) - return out + x - -class ImpalaBlock(nn.Module): - def __init__(self, in_channels, out_channels): - super(ImpalaBlock, self).__init__() - self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1, padding=1) - self.res1 = ResidualBlock(out_channels) - self.res2 = ResidualBlock(out_channels) - - - def forward(self, x): - x = self.conv(x) - x = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)(x) - x = self.res1(x) - x = self.res2(x) - return x - -class ImpalaModel(nn.Module): - def __init__(self,in_channels, recurrent=False, hidden_size=256): - super(ImpalaModel, self).__init__() - self.block1 = ImpalaBlock(in_channels=in_channels, out_channels=8) - self.block2 = ImpalaBlock(in_channels=8, out_channels=16) - self.block3 = ImpalaBlock(in_channels=16, out_channels=16) - self.fc = nn.Linear(in_features=1936, out_features=hidden_size) - - self.output_dim = 256 - self.apply(xavier_uniform_init) - init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init. - constant_(x, 0), nn.init.calculate_gain('relu')) - self.critic_linear = init_(nn.Linear(hidden_size, 1)) - self.train() - - self._recurrent = recurrent - self._hidden_size = hidden_size - - @property - def is_recurrent(self): - return self._recurrent - - @property - def recurrent_hidden_state_size(self): - if self._recurrent: - return self._hidden_size - return 1 - - @property - def output_size(self): - return self._hidden_size - - - def forward(self, x, rnn_hxs, masks): - x = self.block1( x/ 255.0) - x = self.block2(x) - x = self.block3(x) - x = nn.ReLU()(x) - x = Flatten()(x) - x = self.fc(x) - x = nn.ReLU()(x) - - return self.critic_linear(x), x, rnn_hxs - diff --git a/3DOrangeTree/a2c_ppo_acktr/storage.py b/3DOrangeTree/a2c_ppo_acktr/storage.py deleted file mode 100644 index b955e5b..0000000 --- a/3DOrangeTree/a2c_ppo_acktr/storage.py +++ /dev/null @@ -1,482 +0,0 @@ -import torch -from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler - - -def _flatten_helper(T, N, _tensor): - return _tensor.view(T * N, *_tensor.size()[2:]) - - -class RolloutStorage(object): - def __init__(self, num_steps, num_processes, obs_shape, action_space, - recurrent_hidden_state_size, head): - self.obs = torch.zeros(num_steps + 1, num_processes, *obs_shape) - self.num_processes = num_processes - - self.recurrent_hidden_states = torch.zeros( - num_steps + 1, num_processes, recurrent_hidden_state_size) - self.rewards = torch.zeros(num_steps, num_processes, 1) - self.rewardsORIG = torch.zeros(num_steps, num_processes, 1) - - self.value_preds = torch.zeros(num_steps + 1, num_processes, 1) - self.returns = torch.zeros(num_steps + 1, num_processes, 1) - self.action_log_probs = torch.zeros(num_steps, num_processes, 1) - if action_space.__class__.__name__ == 'Discrete': - action_shape = 1 - else: - action_shape = action_space.shape[0] - self.actions = torch.zeros(num_steps, num_processes, action_shape) - if action_space.__class__.__name__ == 'Discrete': - self.actions = self.actions.long() - self.masks = torch.ones(num_steps + 1, num_processes, 1) - - # Masks that indicate whether it's a true terminal state - # or time limit end state - self.bad_masks = torch.ones(num_steps + 1, num_processes, 1) - - self.num_steps = num_steps - self.obs_shape = obs_shape - self.step = 0 - - self.recurrent_hidden_state_size = recurrent_hidden_state_size - self.heads = head - self.success = 16 - self.successTake = 8 - self.hidden_state_size = 256 - - self.obs_batchheadsS = [None] * self.heads - self.r_batchheadsS = [None] * self.heads - self.recurrent_hidden_statesbatchheadsS = [None] * self.heads - self.act_batchheadsS = [None] * self.heads - self.masks_batchheadsS = [None] * self.heads - self.stepheadsS = 0 - - self.obs_batchheadsF = [None] * self.heads - - self.r_batchheadsF = [None] * self.heads - self.recurrent_hidden_statesbatchheadsF = [None] * self.heads - self.act_batchheadsF = [None] * self.heads - self.masks_batchheadsF = [None] * self.heads - self.stepheadsF = 0 - - self.obs_batchS = torch.zeros(self.num_steps + 1, self.success, *self.obs_shape) - self.r_batchS = torch.zeros(self.num_steps, self.success,1) - self.recurrent_hidden_statesS = torch.zeros(self.num_steps + 1, self.success, self.recurrent_hidden_state_size) - self.act_batchS = torch.zeros(self.num_steps, self.success, action_shape) - self.act_batchS = self.act_batchS.long() - self.masks_batchS = torch.zeros(self.num_steps+ 1, self.success, 1) - self.stepS = 0 - self.obs_batchF = torch.zeros(self.num_steps + 1, self.success, *self.obs_shape) - self.r_batchF = torch.zeros(self.num_steps, self.success,1) - self.recurrent_hidden_statesF = torch.zeros(self.num_steps + 1, self.success, self.recurrent_hidden_state_size) - self.act_batchF = torch.zeros(self.num_steps, self.success, action_shape) - self.act_batchF = self.act_batchF.long() - self.masks_batchF = torch.zeros(self.num_steps+ 1, self.success, 1) - self.stepF = 0 - for i in range(self.heads): - self.obs_batchheadsS[i] = torch.zeros(self.num_steps + 1, self.success, *self.obs_shape) - self.r_batchheadsS[i] = torch.zeros(self.num_steps, self.success,1) - self.recurrent_hidden_statesbatchheadsS[i] = torch.zeros(self.num_steps + 1, self.success, self.recurrent_hidden_state_size) - self.act_batchheadsS[i] = torch.zeros(self.num_steps, self.success, action_shape) - self.act_batchheadsS[i] = self.act_batchheadsS[i].long() - self.masks_batchheadsS[i] = torch.zeros(self.num_steps+ 1, self.success, 1) - self.obs_batchheadsF[i] = torch.zeros(self.num_steps + 1, self.success, *self.obs_shape) - self.r_batchheadsF[i] = torch.zeros(self.num_steps, self.success,1) - self.recurrent_hidden_statesbatchheadsF[i] = torch.zeros(self.num_steps + 1, self.success, self.recurrent_hidden_state_size) - self.act_batchheadsF[i] = torch.zeros(self.num_steps, self.success, action_shape) - self.act_batchheadsF[i] = self.act_batchheadsF[i].long() - self.masks_batchheadsF[i] = torch.zeros(self.num_steps+ 1, self.success, 1) - - def contrastvalueReward(self, contrastval): - self.rewardsORIG = torch.clone(self.rewards) - self.rewards = self.rewards + contrastval.unsqueeze(-1) - - def releaseB(self): - return self.obs, self.rewardsORIG, self.recurrent_hidden_states, self.actions, self.masks - def feed_attnRB(self): - obs_batchx, rew_batchx, recurrent_hidden_statesx, act_batchx, masks_batchx = self.releaseB() - obs_batch = obs_batchx[:-1].view(-1, *self.obs.size()[2:]) - obs_batchorig = obs_batchx[:-1] - recurrent_hidden_states_batch = recurrent_hidden_statesx[:-1].view(-1, self.recurrent_hidden_states.size(-1)) - actions_batch = act_batchx.view(-1, self.actions.size(-1)) - masks_batch = masks_batchx[:-1].view(-1, 1) - reward_batch = rew_batchx.squeeze() - return obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch - - def retrievestepS(self): - return self.stepS - def retrieveR(self): - return self.rewards - def retrieveobs(self): - return self.obs - def retrieveeverything(self): - return torch.cat((self.obs_batchS, self.obs_batchF), dim=1), torch.cat((self.r_batchS, self.r_batchF), - dim=1), torch.cat( - (self.masks_batchS, self.masks_batchF), dim=1), torch.cat((self.act_batchS, self.act_batchF), dim=1) - def retrieveRS(self): - return self.r_batchS, self.r_batchF - - - def storeheadsSF(self, head): - self.obs_batchheadsS[head] = self.obs_batchS - self.r_batchheadsS[head] = self.r_batchS - self.recurrent_hidden_statesbatchheadsS[head] = self.recurrent_hidden_statesS - self.act_batchheadsS[head] = self.act_batchS - self.masks_batchheadsS[head] = self.masks_batchS - self.obs_batchheadsF[head] = self.obs_batchF - self.r_batchheadsF[head] = self.r_batchF - self.recurrent_hidden_statesbatchheadsF[head] = self.recurrent_hidden_statesF - self.act_batchheadsF[head] = self.act_batchF - self.masks_batchheadsF[head] = self.masks_batchF - - def addPosNeg(self,ForS,device, args): - totalreward = self.rewards[-20:].sum(0) - if ForS == 1: - rewardssortgood = torch.nonzero(totalreward > 0.5).reshape(-1, ) - indicesrewardbatch = rewardssortgood[ - 0::2] - obsxx = self.obs[:, indicesrewardbatch].to(device) - numberaddedxx = obsxx.shape[1] - if numberaddedxx > 1: - indicesrewardbatch = rewardssortgood[0:4:2] - else: - rewardssortbad = torch.nonzero(totalreward < 0.5).reshape(-1, ) - indicesrewardbatch = rewardssortbad[ 0::2] - obs = self.obs[:, indicesrewardbatch].to(device) - rec = self.recurrent_hidden_states[:, indicesrewardbatch].to(device) - masks = self.masks[:, indicesrewardbatch].to(device) - act = self.actions[:, indicesrewardbatch].to(device) - rew = self.rewards[:, indicesrewardbatch].to(device) - numberadded = obs.shape[1] # number of success obs to be added - - totalreward = self.r_batchS.sum(0) - totalreward = self.r_batchF.sum(0) - # ''' - if numberadded > 0: - if ForS == 1: - numcareabout = self.stepS - elif ForS == 0: - numcareabout = self.stepF - if numberadded + numcareabout <= self.obs_batchS.shape[1]: # i.e. add all the new obs - if ForS == 1: - self.obs_batchS[:, self.stepS:self.stepS + numberadded] = obs - self.r_batchS[:, self.stepS:self.stepS + numberadded] = rew - self.recurrent_hidden_statesS[:, self.stepS:self.stepS + numberadded] = rec - self.act_batchS[:, self.stepS:self.stepS + numberadded] = act - self.masks_batchS[:, self.stepS:self.stepS + numberadded] = masks - self.stepS = (self.stepS + numberadded) - elif ForS == 0: - self.obs_batchF[:, self.stepF:self.stepF + numberadded] = obs - self.r_batchF[:, self.stepF:self.stepF + numberadded] = rew - self.recurrent_hidden_statesF[:, self.stepF:self.stepF + numberadded] = rec - self.act_batchF[:, self.stepF:self.stepF + numberadded] = act - self.masks_batchF[:, self.stepF:self.stepF + numberadded] = masks - self.stepF = (self.stepF + numberadded) - # ''' - elif (numberadded + numcareabout >= self.obs_batchS.shape[1]) and ( - numcareabout < self.obs_batchS.shape[1]): - - if ForS == 1: - numbertoadd = self.obs_batchS.shape[1] - self.stepS - self.obs_batchS[:, self.stepS:self.stepS + numbertoadd, :] = obs[:, :numbertoadd] - self.r_batchS[:, self.stepS:self.stepS + numbertoadd] = rew[:, :numbertoadd] - self.recurrent_hidden_statesS[:, self.stepS:self.stepS + numbertoadd] = rec[:, :numbertoadd] - self.act_batchS[:, self.stepS:self.stepS + numbertoadd] = act[:, :numbertoadd] - self.masks_batchS[:, self.stepS:self.stepS + numbertoadd] = masks[:, :numbertoadd] - self.stepS = (self.stepS + numbertoadd) - elif ForS == 0: - numbertoadd = self.obs_batchS.shape[1] - self.stepF - self.obs_batchF[:, self.stepF:self.stepF + numbertoadd, :] = obs[:, :numbertoadd] - self.r_batchF[:, self.stepF:self.stepF + numbertoadd] = rew[:, :numbertoadd] - self.recurrent_hidden_statesF[:, self.stepF:self.stepF + numbertoadd] = rec[:, :numbertoadd] - self.act_batchF[:, self.stepF:self.stepF + numbertoadd] = act[:, :numbertoadd] - self.masks_batchF[:, self.stepF:self.stepF + numbertoadd] = masks[:, :numbertoadd] - self.stepF = (self.stepF + numbertoadd) - - elif numcareabout == self.obs_batchS.shape[1]: - masks = masks - - if ForS == 1: - lenconsider = obs.shape[1] - self.obs_batchS = torch.cat((self.obs_batchS, obs), - 1) - self.r_batchS = torch.cat((self.r_batchS, rew), - 1) - self.recurrent_hidden_statesS = torch.cat((self.recurrent_hidden_statesS, rec), - 1) - self.act_batchS = torch.cat((self.act_batchS, act), - 1) - self.masks_batchS = torch.cat((self.masks_batchS, masks), - 1) - self.obs_batchS = self.obs_batchS[:, lenconsider:] - self.r_batchS = self.r_batchS[:, lenconsider:] - self.recurrent_hidden_statesS = self.recurrent_hidden_statesS[:, lenconsider:] - self.act_batchS = self.act_batchS[:, lenconsider:] - self.masks_batchS = self.masks_batchS[:, lenconsider:] - - elif ForS == 0: - lenconsider = obs.shape[1] - self.obs_batchF = torch.cat((self.obs_batchF, obs), - 1) - self.r_batchF = torch.cat((self.r_batchF, rew), - 1) - self.recurrent_hidden_statesF = torch.cat((self.recurrent_hidden_statesF, rec), - 1) - self.act_batchF = torch.cat((self.act_batchF, act), - 1) - self.masks_batchF = torch.cat((self.masks_batchF, masks), - 1) - self.obs_batchF = self.obs_batchF[:, lenconsider:] - self.r_batchF = self.r_batchF[:, lenconsider:] - self.recurrent_hidden_statesF = self.recurrent_hidden_statesF[:, lenconsider:] - self.act_batchF = self.act_batchF[:, lenconsider:] - self.masks_batchF = self.masks_batchF[:, lenconsider:] - - - def to(self, device): - self.obs = self.obs.to(device) - self.recurrent_hidden_states = self.recurrent_hidden_states.to(device) - self.rewards = self.rewards.to(device) - self.value_preds = self.value_preds.to(device) - self.returns = self.returns.to(device) - self.action_log_probs = self.action_log_probs.to(device) - self.actions = self.actions.to(device) - self.masks = self.masks.to(device) - self.bad_masks = self.bad_masks.to(device) - - self.obs_batchS = self.obs_batchS.to(device) - self.r_batchS = self.r_batchS.to(device) - self.recurrent_hidden_statesS = self.recurrent_hidden_statesS.to(device) - self.act_batchS = self.act_batchS.to(device) - self.masks_batchS = self.masks_batchS.to(device) - self.obs_batchF = self.obs_batchF.to(device) - self.r_batchF = self.r_batchF.to(device) - self.recurrent_hidden_statesF = self.recurrent_hidden_statesF.to(device) - self.act_batchF = self.act_batchF.to(device) - self.masks_batchF = self.masks_batchF.to(device) - for i in range(self.heads): - self.obs_batchheadsS[i] =self.obs_batchheadsS[i].to(device) - self.r_batchheadsS[i] =self.r_batchheadsS[i].to(device) - self.recurrent_hidden_statesbatchheadsS[i] = self.recurrent_hidden_statesbatchheadsS[i].to(device) - self.act_batchheadsS[i] =self.act_batchheadsS[i].to(device) - self.masks_batchheadsS[i] = self.masks_batchheadsS[i].to(device) - self.obs_batchheadsF[i] = self.obs_batchheadsF[i].to(device) - self.r_batchheadsF[i] = self.r_batchheadsF[i].to(device) - self.recurrent_hidden_statesbatchheadsF[i] = self.recurrent_hidden_statesbatchheadsF[i].to(device) - self.act_batchheadsF[i] = self.act_batchheadsF[i].to(device) - self.masks_batchheadsF[i] = self.masks_batchheadsF[i].to(device) - - def insert(self, obs, recurrent_hidden_states, actions, action_log_probs, - value_preds, rewards, masks, bad_masks, miniN): - self.obs[self.step + 1, miniN].copy_(obs) - self.recurrent_hidden_states[self.step + - 1, miniN].copy_(recurrent_hidden_states) - self.actions[self.step, miniN].copy_(actions) - self.action_log_probs[self.step, miniN].copy_(action_log_probs) - self.value_preds[self.step, miniN].copy_(value_preds) - self.rewards[self.step, miniN].copy_(rewards) - self.masks[self.step + 1, miniN].copy_(masks) - self.bad_masks[self.step + 1, miniN].copy_(bad_masks) - self.step = (self.step + 1) % self.num_steps - - def after_update(self): - self.obs[0].copy_(self.obs[-1]) - self.recurrent_hidden_states[0].copy_(self.recurrent_hidden_states[-1]) - self.masks[0].copy_(self.masks[-1]) - self.bad_masks[0].copy_(self.bad_masks[-1]) - - def compute_returns(self, - next_value, - use_gae, - gamma, - gae_lambda, - use_proper_time_limits=True): - if use_proper_time_limits: - if use_gae: - self.value_preds[-1] = next_value - gae = 0 - for step in reversed(range(self.rewards.size(0))): - delta = self.rewards[step] + gamma * self.value_preds[ - step + 1] * self.masks[step + - 1] - self.value_preds[step] - gae = delta + gamma * gae_lambda * self.masks[step + - 1] * gae - gae = gae * self.bad_masks[step + 1] - self.returns[step] = gae + self.value_preds[step] - else: - self.returns[-1] = next_value - for step in reversed(range(self.rewards.size(0))): - self.returns[step] = (self.returns[step + 1] * \ - gamma * self.masks[step + 1] + self.rewards[step]) * self.bad_masks[step + 1] \ - + (1 - self.bad_masks[step + 1]) * self.value_preds[step] - else: - if use_gae: - self.value_preds[-1] = next_value - gae = 0 - for step in reversed(range(self.rewards.size(0))): - delta = self.rewards[step] + gamma * self.value_preds[ - step + 1] * self.masks[step + - 1] - self.value_preds[step] - gae = delta + gamma * gae_lambda * self.masks[step + - 1] * gae - self.returns[step] = gae + self.value_preds[step] - else: - self.returns[-1] = next_value - for step in reversed(range(self.rewards.size(0))): - self.returns[step] = self.returns[step + 1] * \ - gamma * self.masks[step + 1] + self.rewards[step] - - def feed_attnR(self): - obs_batch = self.obs[:-1].view(-1, *self.obs.size()[2:]) - obs_batchorig = self.obs[:-1] - recurrent_hidden_states_batch = self.recurrent_hidden_states[:-1].view( - -1, self.recurrent_hidden_states.size(-1)) - actions_batch = self.actions.view(-1, - self.actions.size(-1)) - masks_batch = self.masks[:-1].view(-1, 1) - reward_batch = self.rewards.squeeze() - - return obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch - def releaseheadsSF(self, head): - permS = torch.randperm(self.success) - permF = torch.randperm(self.success) - permS = permS[:self.successTake] - permF = permF[:self.successTake] - rew_batch = torch.cat((self.r_batchheadsS[head][:, permS],self.r_batchheadsF[head][:, permF]),dim=1) - obs_batch = torch.cat((self.obs_batchheadsS[head][:, permS],self.obs_batchheadsF[head][:, permF]),dim=1) - recurrent_hidden_states=torch.cat((self.recurrent_hidden_statesbatchheadsS[head][:, permS],self.recurrent_hidden_statesbatchheadsF[head][:, permF]),dim=1) - act_batch=torch.cat((self.act_batchheadsS[head][:, permS],self.act_batchheadsF[head][:, permF]),dim=1) - masks_batch=torch.cat((self.masks_batchheadsS[head][:, permS],self.masks_batchheadsF[head][:, permF]),dim=1) - return obs_batch, rew_batch,recurrent_hidden_states, act_batch, masks_batch - - def releaseSF(self): - permS = torch.randperm(self.success) - permF = torch.randperm(self.success) - permS = permS[:self.successTake] - permF = permF[:self.successTake] - #print(permS) - rew_batch = torch.cat((self.r_batchS[:, permS],self.r_batchF[:, permF]),dim=1) - - totalreward = rew_batch.sum(0) - - obs_batch = torch.cat((self.obs_batchS[:, permS],self.obs_batchF[:, permF]),dim=1) - recurrent_hidden_states=torch.cat((self.recurrent_hidden_statesS[:, permS],self.recurrent_hidden_statesF[:, permF]),dim=1) - act_batch=torch.cat((self.act_batchS[:, permS],self.act_batchF[:, permF]),dim=1) - masks_batch=torch.cat((self.masks_batchS[:, permS],self.masks_batchF[:, permF]),dim=1) - return obs_batch, rew_batch,recurrent_hidden_states, act_batch, masks_batch - - def feed_attnRSF(self): - obs_batchx, rew_batchx, recurrent_hidden_statesx, act_batchx, masks_batchx = self.releaseSF() - obs_batch = obs_batchx[:-1].view(-1, *self.obs.size()[2:]) - obs_batchorig = obs_batchx[:-1] - recurrent_hidden_states_batch = recurrent_hidden_statesx[:-1].view( -1, self.recurrent_hidden_states.size(-1)) - actions_batch = act_batchx.view(-1,self.actions.size(-1)) - masks_batch = masks_batchx[:-1].view(-1, 1) - reward_batch = rew_batchx.squeeze() - return obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch - def feed_attnRSFheads(self,head): - obs_batchx, rew_batchx, recurrent_hidden_statesx, act_batchx, masks_batchx = self.releaseheadsSF(head) - obs_batch = obs_batchx[:-1].view(-1, *self.obs.size()[2:]) - obs_batchorig = obs_batchx[:-1] - recurrent_hidden_states_batch = recurrent_hidden_statesx[:-1].view(-1, self.recurrent_hidden_states.size(-1)) - actions_batch = act_batchx.view(-1, self.actions.size(-1)) - masks_batch = masks_batchx[:-1].view(-1, 1) - reward_batch = rew_batchx.squeeze() - return obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch - - def feed_forward_generator(self, - advantages, - num_mini_batch=None, - mini_batch_size=None): - num_steps, num_processes = self.rewards.size()[0:2] - batch_size = num_processes * num_steps - - - if mini_batch_size is None: - assert batch_size >= num_mini_batch, ( - "PPO requires the number of processes ({}) " - "* number of steps ({}) = {} " - "to be greater than or equal to the number of PPO mini batches ({})." - "".format(num_processes, num_steps, num_processes * num_steps, - num_mini_batch)) - mini_batch_size = batch_size // num_mini_batch - sampler = BatchSampler( - SubsetRandomSampler(range(batch_size)), - mini_batch_size, - drop_last=True) - for indices in sampler: - obs_batch = self.obs[:-1].view(-1, *self.obs.size()[2:])[indices] - recurrent_hidden_states_batch = self.recurrent_hidden_states[:-1].view( - -1, self.recurrent_hidden_states.size(-1))[indices] - actions_batch = self.actions.view(-1, - self.actions.size(-1))[indices] - value_preds_batch = self.value_preds[:-1].view(-1, 1)[indices] - return_batch = self.returns[:-1].view(-1, 1)[indices] - masks_batch = self.masks[:-1].view(-1, 1)[indices] - old_action_log_probs_batch = self.action_log_probs.view(-1, - 1)[indices] - if advantages is None: - adv_targ = None - else: - adv_targ = advantages.view(-1, 1)[indices] - - yield obs_batch, recurrent_hidden_states_batch, actions_batch, \ - value_preds_batch, return_batch, masks_batch, old_action_log_probs_batch, adv_targ - - def recurrent_generator(self, advantages, num_mini_batch): - num_processes = self.rewards.size(1) - assert num_processes >= num_mini_batch, ( - "PPO requires the number of processes ({}) " - "to be greater than or equal to the number of " - "PPO mini batches ({}).".format(num_processes, num_mini_batch)) - num_envs_per_batch = num_processes // num_mini_batch - perm = torch.randperm(num_processes) - for start_ind in range(0, num_processes, num_envs_per_batch): - obs_batch = [] - recurrent_hidden_states_batch = [] - actions_batch = [] - value_preds_batch = [] - return_batch = [] - masks_batch = [] - old_action_log_probs_batch = [] - adv_targ = [] - - for offset in range(num_envs_per_batch): - ind = perm[start_ind + offset] - obs_batch.append(self.obs[:-1, ind]) - recurrent_hidden_states_batch.append( - self.recurrent_hidden_states[0:1, ind]) - actions_batch.append(self.actions[:, ind]) - value_preds_batch.append(self.value_preds[:-1, ind]) - return_batch.append(self.returns[:-1, ind]) - masks_batch.append(self.masks[:-1, ind]) - old_action_log_probs_batch.append( - self.action_log_probs[:, ind]) - adv_targ.append(advantages[:, ind]) - - T, N = self.num_steps, num_envs_per_batch - # These are all tensors of size (T, N, -1) - obs_batch = torch.stack(obs_batch, 1) - actions_batch = torch.stack(actions_batch, 1) - value_preds_batch = torch.stack(value_preds_batch, 1) - return_batch = torch.stack(return_batch, 1) - masks_batch = torch.stack(masks_batch, 1) - old_action_log_probs_batch = torch.stack( - old_action_log_probs_batch, 1) - adv_targ = torch.stack(adv_targ, 1) - - # States is just a (N, -1) tensor - recurrent_hidden_states_batch = torch.stack( - recurrent_hidden_states_batch, 1).view(N, -1) - - # Flatten the (T, N, ...) tensors to (T * N, ...) - obs_batch = _flatten_helper(T, N, obs_batch) - actions_batch = _flatten_helper(T, N, actions_batch) - value_preds_batch = _flatten_helper(T, N, value_preds_batch) - return_batch = _flatten_helper(T, N, return_batch) - masks_batch = _flatten_helper(T, N, masks_batch) - old_action_log_probs_batch = _flatten_helper(T, N, \ - old_action_log_probs_batch) - adv_targ = _flatten_helper(T, N, adv_targ) - - yield obs_batch, recurrent_hidden_states_batch, actions_batch, \ - value_preds_batch, return_batch, masks_batch, old_action_log_probs_batch, adv_targ diff --git a/3DOrangeTree/evaluation.py b/3DOrangeTree/evaluation.py deleted file mode 100644 index 7a77831..0000000 --- a/3DOrangeTree/evaluation.py +++ /dev/null @@ -1,48 +0,0 @@ -import numpy as np -import torch - -from a2c_ppo_acktr import utils -from a2c_ppo_acktr.envs import make_vec_envs - - -def evaluate(actor_critic, obs_rms, env_name, seed, num_processes, eval_log_dir, - device): - eval_envs = make_vec_envs(env_name, seed + num_processes, num_processes, - None, eval_log_dir, device, True) - - vec_norm = utils.get_vec_normalize(eval_envs) - if vec_norm is not None: - vec_norm.eval() - vec_norm.obs_rms = obs_rms - - eval_episode_rewards = [] - - obs = eval_envs.reset() - eval_recurrent_hidden_states = torch.zeros( - num_processes, actor_critic.recurrent_hidden_state_size, device=device) - eval_masks = torch.zeros(num_processes, 1, device=device) - - while len(eval_episode_rewards) < 10: - with torch.no_grad(): - _, action, _, eval_recurrent_hidden_states = actor_critic.act( - obs, - eval_recurrent_hidden_states, - eval_masks, - deterministic=True) - - # Obser reward and next obs - obs, _, done, infos = eval_envs.step(action) - - eval_masks = torch.tensor( - [[0.0] if done_ else [1.0] for done_ in done], - dtype=torch.float32, - device=device) - - for info in infos: - if 'episode' in info.keys(): - eval_episode_rewards.append(info['episode']['r']) - - eval_envs.close() - - print(" Evaluation using {} episodes: mean reward {:.5f}\n".format( - len(eval_episode_rewards), np.mean(eval_episode_rewards))) diff --git a/3DOrangeTree/generate_tmux_yaml.py b/3DOrangeTree/generate_tmux_yaml.py deleted file mode 100644 index 7273286..0000000 --- a/3DOrangeTree/generate_tmux_yaml.py +++ /dev/null @@ -1,37 +0,0 @@ -import argparse - -import yaml - -parser = argparse.ArgumentParser(description='Process some integers.') -parser.add_argument( - '--num-seeds', - type=int, - default=4, - help='number of random seeds to generate') -parser.add_argument( - '--env-names', - default="PongNoFrameskip-v4", - help='environment name separated by semicolons') -args = parser.parse_args() - -ppo_mujoco_template = "python main.py --env-name {0} --algo ppo --use-gae --log-interval 1 --num-steps 2048 --num-processes 1 --lr 3e-4 --entropy-coef 0 --value-loss-coef 0.5 --ppo-epoch 10 --num-mini-batch 32 --gamma 0.99 --tau 0.95 --num-env-steps 1000000 --use-linear-lr-decay --no-cuda --log-dir /tmp/gym/{1}/{1}-{2} --seed {2} --use-proper-time-limits" - -ppo_atari_template = "env CUDA_VISIBLE_DEVICES={2} python main.py --env-name {0} --algo ppo --use-gae --lr 2.5e-4 --clip-param 0.1 --value-loss-coef 0.5 --num-processes 8 --num-steps 128 --num-mini-batch 4 --log-interval 1 --use-linear-lr-decay --entropy-coef 0.01 --log-dir /tmp/gym/{1}/{1}-{2} --seed {2}" - -template = ppo_atari_template - -config = {"session_name": "run-all", "windows": []} - -for i in range(args.num_seeds): - panes_list = [] - for env_name in args.env_names.split(';'): - panes_list.append( - template.format(env_name, - env_name.split('-')[0].lower(), i)) - - config["windows"].append({ - "window_name": "seed-{}".format(i), - "panes": panes_list - }) - -yaml.dump(config, open("run_all.yaml", "w"), default_flow_style=False) diff --git a/3DOrangeTree/gym_unity/__init__.py b/3DOrangeTree/gym_unity/__init__.py deleted file mode 100644 index 5d158f1..0000000 --- a/3DOrangeTree/gym_unity/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Version of the library that will be used to upload to pypi -__version__ = "0.29.0.dev0" - -# Git tag that will be checked to determine whether to trigger upload to pypi -__release_tag__ = None diff --git a/3DOrangeTree/gym_unity/__pycache__/__init__.cpython-36.pyc b/3DOrangeTree/gym_unity/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index 9cd2b99..0000000 Binary files a/3DOrangeTree/gym_unity/__pycache__/__init__.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/gym_unity/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/gym_unity/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index ee35521..0000000 Binary files a/3DOrangeTree/gym_unity/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/gym_unity/envs/__init__.py b/3DOrangeTree/gym_unity/envs/__init__.py deleted file mode 100644 index 322d836..0000000 --- a/3DOrangeTree/gym_unity/envs/__init__.py +++ /dev/null @@ -1,361 +0,0 @@ -import itertools -import numpy as np -from typing import Any, Dict, List, Optional, Tuple, Union - -import gym -from gym import error, spaces - -from mlagents_envs.base_env import ActionTuple, BaseEnv -from mlagents_envs.base_env import DecisionSteps, TerminalSteps -from mlagents_envs import logging_util - - -class UnityGymException(error.Error): - """ - Any error related to the gym wrapper of ml-agents. - """ - - pass - - -logger = logging_util.get_logger(__name__) -logging_util.set_log_level(logging_util.INFO) - -GymStepResult = Tuple[np.ndarray, float, bool, Dict] - - -class UnityToGymWrapper(gym.Env): - """ - Provides Gym wrapper for Unity Learning Environments. - """ - - def __init__( - self, - unity_env: BaseEnv, - uint8_visual: bool = False, - flatten_branched: bool = False, - allow_multiple_obs: bool = False, - action_space_seed: Optional[int] = None, - ): - """ - Environment initialization - :param unity_env: The Unity BaseEnv to be wrapped in the gym. Will be closed when the UnityToGymWrapper closes. - :param uint8_visual: Return visual observations as uint8 (0-255) matrices instead of float (0.0-1.0). - :param flatten_branched: If True, turn branched discrete action spaces into a Discrete space rather than - MultiDiscrete. - :param allow_multiple_obs: If True, return a list of np.ndarrays as observations with the first elements - containing the visual observations and the last element containing the array of vector observations. - If False, returns a single np.ndarray containing either only a single visual observation or the array of - vector observations. - :param action_space_seed: If non-None, will be used to set the random seed on created gym.Space instances. - """ - self._env = unity_env - - # Take a single step so that the brain information will be sent over - if not self._env.behavior_specs: - self._env.step() - - self.visual_obs = None - - # Save the step result from the last time all Agents requested decisions. - self._previous_decision_step: DecisionSteps = None - self._flattener = None - # Hidden flag used by Atari environments to determine if the game is over - self.game_over = False - self._allow_multiple_obs = allow_multiple_obs - - # Check brain configuration - if len(self._env.behavior_specs) != 1: - raise UnityGymException( - "There can only be one behavior in a UnityEnvironment " - "if it is wrapped in a gym." - ) - - self.name = list(self._env.behavior_specs.keys())[0] - self.group_spec = self._env.behavior_specs[self.name] - - if self._get_n_vis_obs() == 0 and self._get_vec_obs_size() == 0: - raise UnityGymException( - "There are no observations provided by the environment." - ) - - if not self._get_n_vis_obs() >= 1 and uint8_visual: - logger.warning( - "uint8_visual was set to true, but visual observations are not in use. " - "This setting will not have any effect." - ) - else: - self.uint8_visual = uint8_visual - if ( - self._get_n_vis_obs() + self._get_vec_obs_size() >= 2 - and not self._allow_multiple_obs - ): - logger.warning( - "The environment contains multiple observations. " - "You must define allow_multiple_obs=True to receive them all. " - "Otherwise, only the first visual observation (or vector observation if" - "there are no visual observations) will be provided in the observation." - ) - - # Check for number of agents in scene. - self._env.reset() - decision_steps, _ = self._env.get_steps(self.name) - self._check_agents(len(decision_steps)) - self._previous_decision_step = decision_steps - - # Set action spaces - if self.group_spec.action_spec.is_discrete(): - self.action_size = self.group_spec.action_spec.discrete_size - branches = self.group_spec.action_spec.discrete_branches - if self.group_spec.action_spec.discrete_size == 1: - self._action_space = spaces.Discrete(branches[0]) - else: - if flatten_branched: - self._flattener = ActionFlattener(branches) - self._action_space = self._flattener.action_space - else: - self._action_space = spaces.MultiDiscrete(branches) - - elif self.group_spec.action_spec.is_continuous(): - if flatten_branched: - logger.warning( - "The environment has a non-discrete action space. It will " - "not be flattened." - ) - - self.action_size = self.group_spec.action_spec.continuous_size - high = np.array([1] * self.group_spec.action_spec.continuous_size) - self._action_space = spaces.Box(-high, high, dtype=np.float32) - else: - raise UnityGymException( - "The gym wrapper does not provide explicit support for both discrete " - "and continuous actions." - ) - - if action_space_seed is not None: - self._action_space.seed(action_space_seed) - - # Set observations space - list_spaces: List[gym.Space] = [] - shapes = self._get_vis_obs_shape() - for shape in shapes: - if uint8_visual: - list_spaces.append(spaces.Box(0, 255, dtype=np.uint8, shape=shape)) - else: - list_spaces.append(spaces.Box(0, 1, dtype=np.float32, shape=shape)) - if self._get_vec_obs_size() > 0: - # vector observation is last - high = np.array([np.inf] * self._get_vec_obs_size()) - list_spaces.append(spaces.Box(-high, high, dtype=np.float32)) - if self._allow_multiple_obs: - self._observation_space = spaces.Tuple(list_spaces) - else: - self._observation_space = list_spaces[0] # only return the first one - - def reset(self) -> Union[List[np.ndarray], np.ndarray]: - """Resets the state of the environment and returns an initial observation. - Returns: observation (object/list): the initial observation of the - space. - """ - self._env.reset() - decision_step, _ = self._env.get_steps(self.name) - n_agents = len(decision_step) - self._check_agents(n_agents) - self.game_over = False - - res: GymStepResult = self._single_step(decision_step) - return res[0] - - def step(self, action: List[Any]) -> GymStepResult: - """Run one timestep of the environment's dynamics. When end of - episode is reached, you are responsible for calling `reset()` - to reset this environment's state. - Accepts an action and returns a tuple (observation, reward, done, info). - Args: - action (object/list): an action provided by the environment - Returns: - observation (object/list): agent's observation of the current environment - reward (float/list) : amount of reward returned after previous action - done (boolean/list): whether the episode has ended. - info (dict): contains auxiliary diagnostic information. - """ - if self.game_over: - raise UnityGymException( - "You are calling 'step()' even though this environment has already " - "returned done = True. You must always call 'reset()' once you " - "receive 'done = True'." - ) - if self._flattener is not None: - # Translate action into list - action = self._flattener.lookup_action(action) - - action = np.array(action).reshape((1, self.action_size)) - - action_tuple = ActionTuple() - if self.group_spec.action_spec.is_continuous(): - action_tuple.add_continuous(action) - else: - action_tuple.add_discrete(action) - self._env.set_actions(self.name, action_tuple) - - self._env.step() - decision_step, terminal_step = self._env.get_steps(self.name) - self._check_agents(max(len(decision_step), len(terminal_step))) - if len(terminal_step) != 0: - # The agent is done - self.game_over = True - return self._single_step(terminal_step) - else: - return self._single_step(decision_step) - - def _single_step(self, info: Union[DecisionSteps, TerminalSteps]) -> GymStepResult: - if self._allow_multiple_obs: - visual_obs = self._get_vis_obs_list(info) - visual_obs_list = [] - for obs in visual_obs: - visual_obs_list.append(self._preprocess_single(obs[0])) - default_observation = visual_obs_list - if self._get_vec_obs_size() >= 1: - default_observation.append(self._get_vector_obs(info)[0, :]) - else: - if self._get_n_vis_obs() >= 1: - visual_obs = self._get_vis_obs_list(info) - default_observation = self._preprocess_single(visual_obs[0][0]) - else: - default_observation = self._get_vector_obs(info)[0, :] - - if self._get_n_vis_obs() >= 1: - visual_obs = self._get_vis_obs_list(info) - self.visual_obs = self._preprocess_single(visual_obs[0][0]) - - done = isinstance(info, TerminalSteps) - - return (default_observation, info.reward[0], done, {"step": info}) - - def _preprocess_single(self, single_visual_obs: np.ndarray) -> np.ndarray: - if self.uint8_visual: - return (255.0 * single_visual_obs).astype(np.uint8) - else: - return single_visual_obs - - def _get_n_vis_obs(self) -> int: - result = 0 - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 3: - result += 1 - return result - - def _get_vis_obs_shape(self) -> List[Tuple]: - result: List[Tuple] = [] - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 3: - result.append(obs_spec.shape) - return result - - def _get_vis_obs_list( - self, step_result: Union[DecisionSteps, TerminalSteps] - ) -> List[np.ndarray]: - result: List[np.ndarray] = [] - for obs in step_result.obs: - if len(obs.shape) == 4: - result.append(obs) - return result - - def _get_vector_obs( - self, step_result: Union[DecisionSteps, TerminalSteps] - ) -> np.ndarray: - result: List[np.ndarray] = [] - for obs in step_result.obs: - if len(obs.shape) == 2: - result.append(obs) - return np.concatenate(result, axis=1) - - def _get_vec_obs_size(self) -> int: - result = 0 - for obs_spec in self.group_spec.observation_specs: - if len(obs_spec.shape) == 1: - result += obs_spec.shape[0] - return result - - def render(self, mode="rgb_array"): - """ - Return the latest visual observations. - Note that it will not render a new frame of the environment. - """ - return self.visual_obs - - def close(self) -> None: - """Override _close in your subclass to perform any necessary cleanup. - Environments will automatically close() themselves when - garbage collected or when the program exits. - """ - self._env.close() - - def seed(self, seed: Any = None) -> None: - """Sets the seed for this env's random number generator(s). - Currently not implemented. - """ - logger.warning("Could not seed environment %s", self.name) - return - - @staticmethod - def _check_agents(n_agents: int) -> None: - if n_agents > 1: - raise UnityGymException( - f"There can only be one Agent in the environment but {n_agents} were detected." - ) - - @property - def metadata(self): - return {"render.modes": ["rgb_array"]} - - @property - def reward_range(self) -> Tuple[float, float]: - return -float("inf"), float("inf") - - @property - def action_space(self) -> gym.Space: - return self._action_space - - @property - def observation_space(self): - return self._observation_space - - -class ActionFlattener: - """ - Flattens branched discrete action spaces into single-branch discrete action spaces. - """ - - def __init__(self, branched_action_space): - """ - Initialize the flattener. - :param branched_action_space: A List containing the sizes of each branch of the action - space, e.g. [2,3,3] for three branches with size 2, 3, and 3 respectively. - """ - self._action_shape = branched_action_space - self.action_lookup = self._create_lookup(self._action_shape) - self.action_space = spaces.Discrete(len(self.action_lookup)) - - @classmethod - def _create_lookup(self, branched_action_space): - """ - Creates a Dict that maps discrete actions (scalars) to branched actions (lists). - Each key in the Dict maps to one unique set of branched actions, and each value - contains the List of branched actions. - """ - possible_vals = [range(_num) for _num in branched_action_space] - all_actions = [list(_action) for _action in itertools.product(*possible_vals)] - # Dict should be faster than List for large action spaces - action_lookup = { - _scalar: _action for (_scalar, _action) in enumerate(all_actions) - } - return action_lookup - - def lookup_action(self, action): - """ - Convert a scalar discrete action into a unique set of branched actions. - :param: action: A scalar value representing one of the discrete actions. - :return: The List containing the branched actions. - """ - return self.action_lookup[action] diff --git a/3DOrangeTree/gym_unity/envs/__pycache__/__init__.cpython-36.pyc b/3DOrangeTree/gym_unity/envs/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index 5dcd4db..0000000 Binary files a/3DOrangeTree/gym_unity/envs/__pycache__/__init__.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/gym_unity/envs/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/gym_unity/envs/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index f0c1c59..0000000 Binary files a/3DOrangeTree/gym_unity/envs/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/gym_unity/tests/test_gym.py b/3DOrangeTree/gym_unity/tests/test_gym.py deleted file mode 100644 index 9f19e52..0000000 --- a/3DOrangeTree/gym_unity/tests/test_gym.py +++ /dev/null @@ -1,277 +0,0 @@ -from unittest import mock -import pytest -import numpy as np - -from gym import spaces -from gym_unity.envs import UnityToGymWrapper -from mlagents_envs.base_env import ( - BehaviorSpec, - ActionSpec, - DecisionSteps, - TerminalSteps, - BehaviorMapping, -) -from mlagents.trainers.tests.dummy_config import create_observation_specs_with_shapes - - -def test_gym_wrapper(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec() - mock_decision_step, mock_terminal_step = create_mock_vector_steps(mock_spec) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - env = UnityToGymWrapper(mock_env) - assert isinstance(env.reset(), np.ndarray) - actions = env.action_space.sample() - assert actions.shape[0] == 2 - obs, rew, done, info = env.step(actions) - assert env.observation_space.contains(obs) - assert isinstance(obs, np.ndarray) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - -def test_branched_flatten(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - vector_action_space_type="discrete", vector_action_space_size=[2, 2, 3] - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, num_agents=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, flatten_branched=True) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 12 - assert env._flattener.lookup_action(0) == [0, 0, 0] - assert env._flattener.lookup_action(11) == [1, 1, 2] - - # Check that False produces a MultiDiscrete - env = UnityToGymWrapper(mock_env, flatten_branched=False) - assert isinstance(env.action_space, spaces.MultiDiscrete) - - -def test_action_space(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - vector_action_space_type="discrete", vector_action_space_size=[5] - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, num_agents=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, flatten_branched=True) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 5 - - env = UnityToGymWrapper(mock_env, flatten_branched=False) - assert isinstance(env.action_space, spaces.Discrete) - assert env.action_space.n == 5 - - -def test_action_space_seed(): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec() - mock_decision_step, mock_terminal_step = create_mock_vector_steps(mock_spec) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - actions = [] - for _ in range(0, 2): - env = UnityToGymWrapper(mock_env, action_space_seed=1337) - env.reset() - actions.append(env.action_space.sample()) - assert (actions[0] == actions[1]).all() - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_visual(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=1, vector_observation_space_size=0 - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8) - assert isinstance(env.observation_space, spaces.Box) - assert isinstance(env.reset(), np.ndarray) - actions = env.action_space.sample() - assert actions.shape[0] == 2 - obs, rew, done, info = env.step(actions) - assert env.observation_space.contains(obs) - assert isinstance(obs, np.ndarray) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_single_visual_and_vector(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=1, - vector_observation_space_size=3, - vector_action_space_size=[2], - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=1 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=True) - assert isinstance(env.observation_space, spaces.Tuple) - assert len(env.observation_space) == 2 - reset_obs = env.reset() - assert isinstance(reset_obs, list) - assert len(reset_obs) == 2 - assert all(isinstance(ob, np.ndarray) for ob in reset_obs) - assert reset_obs[-1].shape == (3,) - assert len(reset_obs[0].shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, list) - assert len(obs) == 2 - assert all(isinstance(ob, np.ndarray) for ob in obs) - assert reset_obs[-1].shape == (3,) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - # check behavior for allow_multiple_obs = False - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=False) - assert isinstance(env.observation_space, spaces.Box) - reset_obs = env.reset() - assert isinstance(reset_obs, np.ndarray) - assert len(reset_obs.shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, np.ndarray) - - -@pytest.mark.parametrize("use_uint8", [True, False], ids=["float", "uint8"]) -def test_gym_wrapper_multi_visual_and_vector(use_uint8): - mock_env = mock.MagicMock() - mock_spec = create_mock_group_spec( - number_visual_observations=2, - vector_observation_space_size=3, - vector_action_space_size=[2], - ) - mock_decision_step, mock_terminal_step = create_mock_vector_steps( - mock_spec, number_visual_observations=2 - ) - setup_mock_unityenvironment( - mock_env, mock_spec, mock_decision_step, mock_terminal_step - ) - - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=True) - assert isinstance(env.observation_space, spaces.Tuple) - assert len(env.observation_space) == 3 - reset_obs = env.reset() - assert isinstance(reset_obs, list) - assert len(reset_obs) == 3 - assert all(isinstance(ob, np.ndarray) for ob in reset_obs) - assert reset_obs[-1].shape == (3,) - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert all(isinstance(ob, np.ndarray) for ob in obs) - assert isinstance(rew, float) - assert isinstance(done, (bool, np.bool_)) - assert isinstance(info, dict) - - # check behavior for allow_multiple_obs = False - env = UnityToGymWrapper(mock_env, uint8_visual=use_uint8, allow_multiple_obs=False) - assert isinstance(env.observation_space, spaces.Box) - reset_obs = env.reset() - assert isinstance(reset_obs, np.ndarray) - assert len(reset_obs.shape) == 3 - actions = env.action_space.sample() - assert actions.shape == (2,) - obs, rew, done, info = env.step(actions) - assert isinstance(obs, np.ndarray) - - -# Helper methods - - -def create_mock_group_spec( - number_visual_observations=0, - vector_action_space_type="continuous", - vector_observation_space_size=3, - vector_action_space_size=None, -): - """ - Creates a mock BrainParameters object with parameters. - """ - # Avoid using mutable object as default param - if vector_action_space_type == "continuous": - if vector_action_space_size is None: - vector_action_space_size = 2 - else: - vector_action_space_size = vector_action_space_size[0] - action_spec = ActionSpec.create_continuous(vector_action_space_size) - else: - if vector_action_space_size is None: - vector_action_space_size = (2,) - else: - vector_action_space_size = tuple(vector_action_space_size) - action_spec = ActionSpec.create_discrete(vector_action_space_size) - obs_shapes = [(vector_observation_space_size,)] - for _ in range(number_visual_observations): - obs_shapes += [(8, 8, 3)] - obs_spec = create_observation_specs_with_shapes(obs_shapes) - return BehaviorSpec(obs_spec, action_spec) - - -def create_mock_vector_steps(specs, num_agents=1, number_visual_observations=0): - """ - Creates a mock BatchedStepResult with vector observations. Imitates constant - vector observations, rewards, dones, and agents. - - :BehaviorSpecs specs: The BehaviorSpecs for this mock - :int num_agents: Number of "agents" to imitate in your BatchedStepResult values. - """ - obs = [np.array([num_agents * [1, 2, 3]], dtype=np.float32).reshape(num_agents, 3)] - if number_visual_observations: - obs += [ - np.zeros(shape=(num_agents, 8, 8, 3), dtype=np.float32) - ] * number_visual_observations - rewards = np.array(num_agents * [1.0]) - agents = np.array(range(0, num_agents)) - group_id = np.array(num_agents * [0]) - group_rewards = np.array(num_agents * [0.0]) - return ( - DecisionSteps(obs, rewards, agents, None, group_id, group_rewards), - TerminalSteps.empty(specs), - ) - - -def setup_mock_unityenvironment(mock_env, mock_spec, mock_decision, mock_termination): - """ - Takes a mock UnityEnvironment and adds the appropriate properties, defined by the mock - GroupSpec and BatchedStepResult. - - :Mock mock_env: A mock UnityEnvironment, usually empty. - :Mock mock_spec: An AgentGroupSpec object that specifies the params of this environment. - :Mock mock_decision: A DecisionSteps object that will be returned at each step and reset. - :Mock mock_termination: A TerminationSteps object that will be returned at each step and reset. - """ - mock_env.behavior_specs = BehaviorMapping({"MockBrain": mock_spec}) - mock_env.get_steps.return_value = (mock_decision, mock_termination) diff --git a/3DOrangeTree/main.py b/3DOrangeTree/main.py deleted file mode 100644 index b737f45..0000000 --- a/3DOrangeTree/main.py +++ /dev/null @@ -1,236 +0,0 @@ -import copy -import glob -import os -import time -from collections import deque - -import gym -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - -from a2c_ppo_acktr import algo, utils -from a2c_ppo_acktr.algo import gail -from a2c_ppo_acktr.arguments import get_args -from a2c_ppo_acktr.modelRL import Policy -from a2c_ppo_acktr.modelConspec import PolicyCL - -from Utils.utils import plot_learning_curve -from mlagents_envs.environment import UnityEnvironment -from gym_unity.envs import UnityToGymWrapper -import numpy as np - -from a2c_ppo_acktr.storage import RolloutStorage -from evaluation import evaluate -from moduleConspec import moduleCL -import traceback - - -def main(): - args = get_args() - - keysUsed = np.zeros((args.head,)) - goodones = np.zeros((args.head,)) - - torch.manual_seed(args.seed) - torch.cuda.manual_seed_all(args.seed) - - if args.cuda and torch.cuda.is_available() and args.cuda_deterministic: - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - - log_dir = os.path.expanduser(args.log_dir) - eval_log_dir = log_dir + "_eval" - utils.cleanup_log_dir(log_dir) - utils.cleanup_log_dir(eval_log_dir) - - torch.set_num_threads(1) - device = torch.device("cuda:0" if args.cuda else "cpu") - #chmod -R 755 /home/chen/PycharmProjects/ProjTreeSimplePink2extrahardLinux4590_338084842miss/GridWorld.x86_64 - envU = UnityEnvironment(r"[directory...]/ProjTreeSimplePink2extrahardLinux4590_338084842miss/GridWorld.x86_64",seed=2,worker_id=3,no_graphics=False) - envs = UnityToGymWrapper(envU, allow_multiple_obs=True) - obsspace = (3, 84, 84) - - actor_critic = Policy( - obsspace, - envs.action_space, - base_kwargs={'recurrent': args.recurrent_policy}) - actor_critic.to(device) - - actor_criticCL = PolicyCL( - obsspace, - envs.action_space, - base_kwargs={'recurrent': args.recurrent_policy}) - actor_criticCL.to(device) - save_path = args.save_dir - - moduleuse = moduleCL(input_size=256, hidden_size=100, head=args.head, device=device, args=args) - moduleuse.to(device) - - keysUsedt = torch.FloatTensor(keysUsed).to(device=device) - - if args.algo == 'Conspec': - agent = algo.PPOConspec( - actor_critic,actor_criticCL, - args.clip_param, - args.ppo_epoch, - args.num_mini_batch, - args.value_loss_coef, - args.entropy_coef, - moduleuse, - args.choiceCLparams, - args, - lrCL=args.lrCL, - lr=args.lr, - eps=args.eps, - max_grad_norm=args.max_grad_norm) - - - rollouts = RolloutStorage(args.num_steps, args.num_processes, - obsspace, envs.action_space, - actor_critic.recurrent_hidden_state_size, args.head) # envs.observation_space.shape - - minibsize = args.num_processes - rollouts.to(device) - - episode_rewards = deque(maxlen=10) - - start = time.time() - num_updates = int( - args.num_env_steps) // args.num_steps // args.num_processes - - - for j in range(1002): - - for kkk in range(minibsize): - obs = envs.reset() - obs = torch.from_numpy(obs[0]).permute((2,0,1)).to(device) - rollouts.obs[0,kkk].copy_(obs) - donetotal = 0 - done = 0 - maskstotal = torch.ones((args.num_processes, 1)) # .to(device) - - if args.use_linear_lr_decay: - # decrease learning rate linearly - utils.update_linear_schedule( - agent.optimizer, j, num_updates, - agent.optimizer.lr if args.algo == "acktr" else args.lr) - - for step in range(args.num_steps): - - if done > 0: - if donetotal < 2: - donetotal += 1 - obs = envs.reset() - obs = torch.from_numpy(obs[0]).permute((2, 0, 1)).to(device)*255. - rollouts.obs[step, kkk].copy_(obs) - elif donetotal == 2: - donetotal += 1 - - # Sample actions - with torch.no_grad(): - value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( - rollouts.obs[step,kkk].unsqueeze(0).tile((minibsize,1,1,1)), rollouts.recurrent_hidden_states[step,kkk].reshape(-1,1).tile((minibsize,1)), - rollouts.masks[step,kkk].reshape(-1,1).tile((minibsize,1))) - - if donetotal < 2.5: - obs, reward, done, infos = envs.step(action[0].detach().cpu().numpy()) - obs = torch.from_numpy(obs[0]).unsqueeze(0).permute((0, 3, 1, 2)).to(device)*255. - - reward = np.floor(reward / 5.) / 10. #normalize rewards since they were arbitrarily set at 50 - reward = torch.from_numpy((reward).reshape(-1, 1)).reshape(-1, 1) - done = np.ones((1,)) * done - reward = reward * maskstotal - maskstotalobs = torch.tile( - torch.unsqueeze(torch.unsqueeze((maskstotal), -1), -1), - (1, obs.shape[1], obs.shape[2], obs.shape[3])) - obs = obs * maskstotalobs.to(device) - masks = torch.FloatTensor( - [[1.0] if done_ else [1.0] for done_ in done]) - maskstotal = masks - bad_masks = maskstotal - else: - obs = obs * 0. - done = 0 - - - rollouts.insert(obs[0], recurrent_hidden_states[0], action[0], - action_log_prob[0], value[0], reward[0], masks[0], bad_masks[0], kkk) - try: - with torch.no_grad(): - next_value = actor_critic.get_value( - rollouts.obs[-1], rollouts.recurrent_hidden_states[-1], - rollouts.masks[-1]).detach() - - ############################### - with torch.no_grad(): - rewardstotal = rollouts.retrieveR() - episode_rewards.append(rewardstotal[-20:].sum(0).mean().cpu().detach().numpy()) - rollouts.addPosNeg(1, device, args) - rollouts.addPosNeg(0, device, args) - agent.fictitiousReward(rollouts, keysUsedt, device, j) - - ############################### - - rollouts.compute_returns(next_value, args.use_gae, args.gamma, - args.gae_lambda, args.use_proper_time_limits) - - value_loss, action_loss, dist_entropy, keysUsed, goodones = agent.update(rollouts, args.head, keysUsed, - goodones, j) - keysUsedt = torch.FloatTensor(keysUsed).to(device=device) #for the next fictitius reward - - if j % args.log_interval == 0 and len(episode_rewards) > 1: - total_num_steps = (j + 1) * args.num_processes * args.num_steps - end = time.time() - print( - "Updates {}, num timesteps {}, FPS {} \n Last {} training episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}\n" - .format(j, total_num_steps, - int(total_num_steps / (end - start)), - len(episode_rewards), np.mean(episode_rewards), - np.median(episode_rewards), np.min(episode_rewards), - np.max(episode_rewards), dist_entropy, value_loss, - action_loss)) - LOGFILE = './Results' + str(args.algo) + 'HEAD' + str(args.head)+ 'choiceCLparam' + str( - args.choiceCLparams) + 'lrCL' + str(args.lrCL) + 'seed' + str(args.seed)+ 'factor' + str(args.factorR) + '.txt' - print(LOGFILE) - printlog1 = f' results {total_num_steps} {episode_rewards[-1]} \n' - with open(LOGFILE, 'a') as f: - f.write(printlog1) - - if (args.eval_interval is not None and len(episode_rewards) > 1 - and j % args.eval_interval == 0): # a - obs_rms = utils.get_vec_normalize(envs).obs_rms - evaluate(actor_critic, obs_rms, args.env_name, args.seed, - args.num_processes, eval_log_dir, device) - - - # ''' - if (j > 700) and (j % 50 ==0): # or (j > 998): # and (j > 500): - torch.save(actor_critic, - os.path.join(save_path, - "model" + str(args.algo) + 'HEAD' + str(args.head)+ 'factor' + str(args.factorR) + str( - args.seed) + 'seed' + str(j) + ".pt")) - torch.save(actor_criticCL, - os.path.join(save_path, - "modelCL" + str(args.algo) + 'HEAD' + str(args.head)+ 'factor' + str(args.factorR) + str( - args.seed) + 'seed' + str(j) + ".pt")) - torch.save(moduleuse, - os.path.join(save_path, "moduleuse" + 'HEAD' + str(args.head)+ str(args.algo) + 'factor' + str( - args.factorR) + str( - args.seed) + 'seed' + str(j) + ".pt")) - - - - except: - LOGDIR = './' - LOGFILE = LOGDIR + 'traceback.txt' - printlog = f'It: {traceback.format_exc()} \n' - with open(LOGFILE, 'a') as f: - f.write(printlog) - - - -if __name__ == "__main__": - main() diff --git a/3DOrangeTree/maintestBlackobj.py b/3DOrangeTree/maintestBlackobj.py deleted file mode 100644 index f4866c2..0000000 --- a/3DOrangeTree/maintestBlackobj.py +++ /dev/null @@ -1,232 +0,0 @@ -import copy -import glob -import os -import time -from collections import deque - -import gym -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - -from a2c_ppo_acktr import algo, utils -from a2c_ppo_acktr.algo import gail -from a2c_ppo_acktr.arguments import get_args -from a2c_ppo_acktr.modelRL import Policy -from a2c_ppo_acktr.modelConspec import PolicyCL - -from Utils.utils import plot_learning_curve -from mlagents_envs.environment import UnityEnvironment -from gym_unity.envs import UnityToGymWrapper -import numpy as np - -from a2c_ppo_acktr.storage import RolloutStorage -from evaluation import evaluate -from moduleConspec import moduleCL -import traceback - - - - - -def main(): - args = get_args() - - keysUsed = np.zeros((args.head,)) - goodones = np.zeros((args.head,)) - - torch.manual_seed(args.seed) - torch.cuda.manual_seed_all(args.seed) - - if args.cuda and torch.cuda.is_available() and args.cuda_deterministic: - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - - log_dir = os.path.expanduser(args.log_dir) - eval_log_dir = log_dir + "_eval" - utils.cleanup_log_dir(log_dir) - utils.cleanup_log_dir(eval_log_dir) - - torch.set_num_threads(1) - device = torch.device("cuda:0" if args.cuda else "cpu") - #chmod -R 755 /home/chen/PycharmProjects/ProjTreeSimplePink2extrahardLinux4590_33_84842onevarb/GridWorld.x86_64 - envU = UnityEnvironment(r"[directory...]/ProjTreeSimplePink2extrahardLinux4590_33_84842onevarb/GridWorld.x86_64",seed=2,worker_id=1,no_graphics=False) #ProjTreeSimplePink2extrahardLinux4590_338084842Testmissb - - envs = UnityToGymWrapper(envU, allow_multiple_obs=True) - obsspace = (3, 84, 84) - - actor_critic = Policy( - obsspace, - envs.action_space, - base_kwargs={'recurrent': args.recurrent_policy}) # envs.observation_space.shape, - actor_critic.to(device) - - actor_criticCL = PolicyCL( - obsspace, - envs.action_space, - base_kwargs={'recurrent': args.recurrent_policy}) # envs.observation_space.shape, - actor_criticCL.to(device) - save_path = args.save_dir - - moduleuse = moduleCL(input_size=256, hidden_size=100, head=args.head, device=device, args=args) - moduleuse.to(device) - actor_criticCL = torch.load(os.path.join(save_path, "modelCL" + str(args.algo) + 'HEAD' + str( - args.head) + 'factor' + str(args.factorR) + str( - args.seed) + "seed1000.pt")) # modelCLxxxxOneshot35seed776 modelCLxxxxOneshot31seed385 modelCLxxxxOneshot32seed1499 modelCLxxxxOneshot33seed860 #modelCLxxxxOneshot34seed941 - actor_critic = torch.load(os.path.join(save_path, "model" + str(args.algo) + 'HEAD' + str( - args.head) + 'factor' + str(args.factorR) + str( - args.seed) + "seed1000.pt")) # modelxxxxOneshot35seed776 modelxxxxOneshot31seed385 modelxxxxOneshot32seed1499 modelxxxxOneshot33seed860 #modelxxxxOneshot34seed941 - moduleuse = torch.load(os.path.join(save_path, "moduleuse" + 'HEAD' + str(args.head) + str( - args.algo) + 'factor' + str(args.factorR) + str(args.seed) + "seed1000.pt")) - - - if args.algo == 'Conspec': - agent = algo.PPOConspectest( - actor_critic,actor_criticCL, - args.clip_param, - args.ppo_epoch, - args.num_mini_batch, - args.value_loss_coef, - args.entropy_coef, - moduleuse, - args.choiceCLparams, - args, - lrCL=args.lrCL, - lr=args.lr, - eps=args.eps, - max_grad_norm=args.max_grad_norm) - - - rollouts = RolloutStorage(args.num_steps, args.num_processes, - obsspace, envs.action_space, - actor_critic.recurrent_hidden_state_size, args.head) # envs.observation_space.shape - - minibsize = args.num_processes - - rollouts.to(device) - - - episode_rewards = deque(maxlen=10) - - start = time.time() - num_updates = int( - args.num_env_steps) // args.num_steps // args.num_processes - - quota = 0 - - for j in range(10): - for kkk in range(minibsize): - obs = envs.reset() - obs = torch.from_numpy(obs[0]).permute((2,0,1)).to(device) - rollouts.obs[0,kkk].copy_(obs) - donetotal = 0 - done = 0 - maskstotal = torch.ones((args.num_processes, 1)) # .to(device) - - if args.use_linear_lr_decay: - # decrease learning rate linearly - utils.update_linear_schedule( - agent.optimizer, j, num_updates, - agent.optimizer.lr if args.algo == "acktr" else args.lr) - - for step in range(args.num_steps): - - if done > 0: - if donetotal < 2: - donetotal += 1 - obs = envs.reset() - obs = torch.from_numpy(obs[0]).permute((2, 0, 1)).to(device)*255. - rollouts.obs[step, kkk].copy_(obs) - elif donetotal == 2: - donetotal += 1 - - # Sample actions - with torch.no_grad(): - value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( - rollouts.obs[step,kkk].unsqueeze(0).tile((minibsize,1,1,1)), rollouts.recurrent_hidden_states[step,kkk].reshape(-1,1).tile((minibsize,1)), - rollouts.masks[step,kkk].reshape(-1,1).tile((minibsize,1))) - - if donetotal < 2.5: - obs, reward, done, infos = envs.step(action[0].detach().cpu().numpy()) - obs = torch.from_numpy(obs[0]).unsqueeze(0).permute((0, 3, 1, 2)).to(device)*255. - - reward = np.floor(reward / 5.) / 10. - reward = torch.from_numpy((reward).reshape(-1, 1)).reshape(-1, 1) ##################### - done = np.ones((1,)) * done - reward = reward * maskstotal - maskstotalobs = torch.tile( - torch.unsqueeze(torch.unsqueeze((maskstotal), -1), -1), - (1, obs.shape[1], obs.shape[2], obs.shape[3])) - obs = obs * maskstotalobs.to(device) - masks = torch.FloatTensor( - [[1.0] if done_ else [1.0] for done_ in done]) - maskstotal = masks - bad_masks = maskstotal - else: - obs = obs * 0. - done = 0 - - - rollouts.insert(obs[0], recurrent_hidden_states[0], action[0], - action_log_prob[0], value[0], reward[0], masks[0], bad_masks[0], kkk) - try: - with torch.no_grad(): - next_value = actor_critic.get_value( - rollouts.obs[-1], rollouts.recurrent_hidden_states[-1], - rollouts.masks[-1]).detach() - - ############################### - with torch.no_grad(): - rewardstotal = rollouts.retrieveR() - episode_rewards.append(rewardstotal[-22:].sum(0).mean().cpu().detach().numpy()) - - rollouts.addPosNeg(1, device, args) - rollouts.addPosNeg(0, device, args) - - - ############################### - - rollouts.compute_returns(next_value, args.use_gae, args.gamma, - args.gae_lambda, args.use_proper_time_limits) - - if j % args.log_interval == 0 and len(episode_rewards) > 0: - total_num_steps = (j + 1) * args.num_processes * args.num_steps - end = time.time() - print( - "Updates {}, num timesteps {}, FPS {} \n Last {} training episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}\n" - .format(j, total_num_steps, - int(total_num_steps / (end - start)), - len(episode_rewards), np.mean(episode_rewards), - np.median(episode_rewards), np.min(episode_rewards), - np.max(episode_rewards), 0, 0, - 0)) - LOGFILE = './Results_blackobj' + str(args.env_name) + 'choiceCLparam' + str( - args.choiceCLparams) + 'lrCL' + str(args.lrCL) + 'factor' + str(args.factorR) + 'seed' + str(args.seed) + '.txt' - print(LOGFILE) - printlog1 = f' results {total_num_steps} {episode_rewards[-1]} \n' - with open(LOGFILE, 'a') as f: - f.write(printlog1) - if (args.eval_interval is not None and len(episode_rewards) > 1 - and j % args.eval_interval == 0): # a - obs_rms = utils.get_vec_normalize(envs).obs_rms - evaluate(actor_critic, obs_rms, args.env_name, args.seed, - args.num_processes, eval_log_dir, device) - - - - - - - except: - LOGDIR = './' - LOGFILE = LOGDIR + 'traceback.txt' - printlog = f'It: {traceback.format_exc()} \n' - with open(LOGFILE, 'a') as f: - f.write(printlog) - - - -if __name__ == "__main__": - main() diff --git a/3DOrangeTree/maintestNewContext.py b/3DOrangeTree/maintestNewContext.py deleted file mode 100644 index 836d46f..0000000 --- a/3DOrangeTree/maintestNewContext.py +++ /dev/null @@ -1,232 +0,0 @@ -import copy -import glob -import os -import time -from collections import deque - -import gym -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - -from a2c_ppo_acktr import algo, utils -from a2c_ppo_acktr.algo import gail -from a2c_ppo_acktr.arguments import get_args -from a2c_ppo_acktr.modelRL import Policy -from a2c_ppo_acktr.modelConspec import PolicyCL - -from Utils.utils import plot_learning_curve -from mlagents_envs.environment import UnityEnvironment -from gym_unity.envs import UnityToGymWrapper -import numpy as np - -from a2c_ppo_acktr.storage import RolloutStorage -from evaluation import evaluate -from moduleConspec import moduleCL -import traceback - - - - -def main(): - args = get_args() - - keysUsed = np.ones((args.head,)) - goodones = np.zeros((args.head,)) - - torch.manual_seed(args.seed) - torch.cuda.manual_seed_all(args.seed) - - if args.cuda and torch.cuda.is_available() and args.cuda_deterministic: - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - - log_dir = os.path.expanduser(args.log_dir) - eval_log_dir = log_dir + "_eval" - utils.cleanup_log_dir(log_dir) - utils.cleanup_log_dir(eval_log_dir) - - torch.set_num_threads(1) - device = torch.device("cuda:0" if args.cuda else "cpu") - #chmod -R 755 /home/chen/PycharmProjects/ProjTreeSimplePink2extrahardLinux4590_338084842TestmissRoommgp/GridWorld.x86_64 - envU = UnityEnvironment(r"[directory...]/ProjTreeSimplePink2extrahardLinux4590_338084842TestmissRoommgp/GridWorld.x86_64",seed=2,worker_id=1,no_graphics=False) #ProjTreeSimplePink2extrahardLinux4590_338084842Testmissb - - envs = UnityToGymWrapper(envU, allow_multiple_obs=True) - obsspace = (3, 84, 84) - - actor_critic = Policy( - obsspace, - envs.action_space, - base_kwargs={'recurrent': args.recurrent_policy}) # envs.observation_space.shape, - actor_critic.to(device) - - actor_criticCL = PolicyCL( - obsspace, - envs.action_space, - base_kwargs={'recurrent': args.recurrent_policy}) # envs.observation_space.shape, - actor_criticCL.to(device) - save_path = args.save_dir - - moduleuse = moduleCL(input_size=256, hidden_size=100, head=args.head, device=device, args=args) - moduleuse.to(device) - actor_criticCL = torch.load(os.path.join(save_path, "modelCL"+ str(args.algo ) + 'HEAD' + str(args.head)+ 'factor' + str(args.factorR) + str(args.seed) + "seed1000.pt"))#modelCLxxxxOneshot35seed776 modelCLxxxxOneshot31seed385 modelCLxxxxOneshot32seed1499 modelCLxxxxOneshot33seed860 #modelCLxxxxOneshot34seed941 - actor_critic = torch.load(os.path.join(save_path, "model" + str(args.algo ) + 'HEAD' + str(args.head)+ 'factor' + str(args.factorR) + str(args.seed) + "seed1000.pt"))#modelxxxxOneshot35seed776 modelxxxxOneshot31seed385 modelxxxxOneshot32seed1499 modelxxxxOneshot33seed860 #modelxxxxOneshot34seed941 - moduleuse = torch.load(os.path.join(save_path, "moduleuse" + 'HEAD' + str(args.head) + str(args.algo) + 'factor' + str(args.factorR) + str(args.seed) + "seed1000.pt")) - keysUsedt = torch.FloatTensor(keysUsed).to(device=device) - - if args.algo == 'Conspec': - agent = algo.PPOConspectest( - actor_critic,actor_criticCL, - args.clip_param, - args.ppo_epoch, - args.num_mini_batch, - args.value_loss_coef, - args.entropy_coef, - moduleuse, - args.choiceCLparams, - args, - lrCL=args.lrCL, - lr=args.lr, - eps=args.eps, - max_grad_norm=args.max_grad_norm) - - - rollouts = RolloutStorage(args.num_steps, args.num_processes, - obsspace, envs.action_space, - actor_critic.recurrent_hidden_state_size, args.head) # envs.observation_space.shape - - minibsize = args.num_processes - - rollouts.to(device) - - episode_rewards = deque(maxlen=10) - - start = time.time() - num_updates = int( - args.num_env_steps) // args.num_steps // args.num_processes - - quota = 0 - - for j in range(20): - - for kkk in range(minibsize): - obs = envs.reset() - obs = torch.from_numpy(obs[0]).permute((2,0,1)).to(device) - rollouts.obs[0,kkk].copy_(obs) - donetotal = 0 - done = 0 - maskstotal = torch.ones((args.num_processes, 1)) # .to(device) - - - if args.use_linear_lr_decay: - # decrease learning rate linearly - utils.update_linear_schedule( - agent.optimizer, j, num_updates, - agent.optimizer.lr if args.algo == "acktr" else args.lr) - - for step in range(args.num_steps): - - if done > 0: - if donetotal < 2: - donetotal += 1 - obs = envs.reset() - obs = torch.from_numpy(obs[0]).permute((2, 0, 1)).to(device)*255. - rollouts.obs[step, kkk].copy_(obs) - elif donetotal == 2: - donetotal += 1 - - # Sample actions - with torch.no_grad(): - - value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( - rollouts.obs[step,kkk].unsqueeze(0).tile((minibsize,1,1,1)), rollouts.recurrent_hidden_states[step,kkk].reshape(-1,1).tile((minibsize,1)), - rollouts.masks[step,kkk].reshape(-1,1).tile((minibsize,1))) - - if donetotal < 2.5: - obs, reward, done, infos = envs.step(action[0].detach().cpu().numpy()) - obs = torch.from_numpy(obs[0]).unsqueeze(0).permute((0, 3, 1, 2)).to(device)*255. - - reward = np.floor(reward / 5.) / 10. - reward = torch.from_numpy((reward).reshape(-1, 1)).reshape(-1, 1) ##################### - done = np.ones((1,)) * done ##################### - reward = reward * maskstotal # * 1./1000. - maskstotalobs = torch.tile( - torch.unsqueeze(torch.unsqueeze((maskstotal), -1), -1), - (1, obs.shape[1], obs.shape[2], obs.shape[3])) - obs = obs * maskstotalobs.to(device) - masks = torch.FloatTensor( - [[1.0] if done_ else [1.0] for done_ in done]) - maskstotal = masks - bad_masks = maskstotal - else: - obs = obs * 0. - done = 0 - - - rollouts.insert(obs[0], recurrent_hidden_states[0], action[0], - action_log_prob[0], value[0], reward[0], masks[0], bad_masks[0], kkk) - try: - with torch.no_grad(): - next_value = actor_critic.get_value( - rollouts.obs[-1], rollouts.recurrent_hidden_states[-1], - rollouts.masks[-1]).detach() - - ############################### - with torch.no_grad(): - # now compute new rewards - rewardstotal = rollouts.retrieveR() - episode_rewards.append(rewardstotal[-22:].sum(0).mean().cpu().detach().numpy()) - - rollouts.addPosNeg(1, device, args) - rollouts.addPosNeg(0, device, args) - - agent.fictitiousReward(rollouts, keysUsedt, device, j) - - ############################### - - rollouts.compute_returns(next_value, args.use_gae, args.gamma, - args.gae_lambda, args.use_proper_time_limits) - - value_loss, action_loss, dist_entropy, keysUsed, goodones = agent.update(rollouts, args.head, keysUsed, - goodones, j) - keysUsedt = torch.FloatTensor(keysUsed).to(device=device) #for the next fictitius reward - - if j % args.log_interval == 0 and len(episode_rewards) > 0: - total_num_steps = (j + 1) * args.num_processes * args.num_steps - end = time.time() - print( - "Updates {}, num timesteps {}, FPS {} \n Last {} training episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}\n" - .format(j, total_num_steps, - int(total_num_steps / (end - start)), - len(episode_rewards), np.mean(episode_rewards), - np.median(episode_rewards), np.min(episode_rewards), - np.max(episode_rewards), 0, 0, - 0)) - LOGFILE = './ResultsNewContext' + str(args.env_name) + 'choiceCLparam' + str( - args.choiceCLparams) + 'lrCL' + str(args.lrCL)+ 'factor' + str(args.factorR) + 'seed' + str(args.seed) + '.txt' - print(LOGFILE) - printlog1 = f' results {total_num_steps} {episode_rewards[-1]} \n' - with open(LOGFILE, 'a') as f: - f.write(printlog1) - - if (args.eval_interval is not None and len(episode_rewards) > 1 - and j % args.eval_interval == 0): # a - obs_rms = utils.get_vec_normalize(envs).obs_rms - evaluate(actor_critic, obs_rms, args.env_name, args.seed, - args.num_processes, eval_log_dir, device) - - - - - except: - LOGDIR = './' - LOGFILE = LOGDIR + 'traceback.txt' - printlog = f'It: {traceback.format_exc()} \n' - with open(LOGFILE, 'a') as f: - f.write(printlog) - - -if __name__ == "__main__": - main() diff --git a/3DOrangeTree/misc_util.py b/3DOrangeTree/misc_util.py deleted file mode 100644 index c0a4238..0000000 --- a/3DOrangeTree/misc_util.py +++ /dev/null @@ -1,41 +0,0 @@ -import numpy as np -import random -import gym -import torch -import torch.nn as nn - - -def set_global_seeds(seed): - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - - -def set_global_log_levels(level): - gym.logger.set_level(level) - - -def orthogonal_init(module, gain=nn.init.calculate_gain('relu')): - if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): - nn.init.orthogonal_(module.weight.data, gain) - nn.init.constant_(module.bias.data, 0) - return module - - -def xavier_uniform_init(module, gain=1.0): - if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): - nn.init.xavier_uniform_(module.weight.data, gain) - nn.init.constant_(module.bias.data, 0) - return module - - -def adjust_lr(optimizer, init_lr, timesteps, max_timesteps): - lr = init_lr * (1 - (timesteps / max_timesteps)) - for param_group in optimizer.param_groups: - param_group['lr'] = lr - return optimizer - - -def get_n_params(model): - return str(np.round(np.array([p.numel() for p in model.parameters()]).sum() / 1e6, 3)) + ' M params' \ No newline at end of file diff --git a/3DOrangeTree/mlagents_envs/__init__.py b/3DOrangeTree/mlagents_envs/__init__.py deleted file mode 100644 index 5d158f1..0000000 --- a/3DOrangeTree/mlagents_envs/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Version of the library that will be used to upload to pypi -__version__ = "0.29.0.dev0" - -# Git tag that will be checked to determine whether to trigger upload to pypi -__release_tag__ = None diff --git a/3DOrangeTree/mlagents_envs/__pycache__/__init__.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index 424b8f7..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/__init__.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index c5467df..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/base_env.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/base_env.cpython-36.pyc deleted file mode 100644 index 8499cc7..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/base_env.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/base_env.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/base_env.cpython-37.pyc deleted file mode 100644 index 3e6775b..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/base_env.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/communicator.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/communicator.cpython-36.pyc deleted file mode 100644 index 0bbb1d4..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/communicator.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/communicator.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/communicator.cpython-37.pyc deleted file mode 100644 index c0408f9..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/communicator.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/env_utils.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/env_utils.cpython-36.pyc deleted file mode 100644 index 1522231..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/env_utils.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/env_utils.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/env_utils.cpython-37.pyc deleted file mode 100644 index 98d760a..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/env_utils.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/environment.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/environment.cpython-36.pyc deleted file mode 100644 index a345498..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/environment.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/environment.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/environment.cpython-37.pyc deleted file mode 100644 index 13ed5c2..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/environment.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/exception.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/exception.cpython-36.pyc deleted file mode 100644 index d3962d2..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/exception.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/exception.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/exception.cpython-37.pyc deleted file mode 100644 index d626459..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/exception.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/logging_util.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/logging_util.cpython-36.pyc deleted file mode 100644 index 26c6e7f..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/logging_util.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/logging_util.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/logging_util.cpython-37.pyc deleted file mode 100644 index 4986d38..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/logging_util.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/rpc_communicator.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/rpc_communicator.cpython-36.pyc deleted file mode 100644 index 7154af8..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/rpc_communicator.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/rpc_communicator.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/rpc_communicator.cpython-37.pyc deleted file mode 100644 index 341623f..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/rpc_communicator.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/rpc_utils.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/rpc_utils.cpython-36.pyc deleted file mode 100644 index c3062a0..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/rpc_utils.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/rpc_utils.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/rpc_utils.cpython-37.pyc deleted file mode 100644 index 9852ff5..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/rpc_utils.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/timers.cpython-36.pyc b/3DOrangeTree/mlagents_envs/__pycache__/timers.cpython-36.pyc deleted file mode 100644 index 9a0c362..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/timers.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/__pycache__/timers.cpython-37.pyc b/3DOrangeTree/mlagents_envs/__pycache__/timers.cpython-37.pyc deleted file mode 100644 index c2a1a83..0000000 Binary files a/3DOrangeTree/mlagents_envs/__pycache__/timers.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/base_env.py b/3DOrangeTree/mlagents_envs/base_env.py deleted file mode 100644 index 02edc4e..0000000 --- a/3DOrangeTree/mlagents_envs/base_env.py +++ /dev/null @@ -1,616 +0,0 @@ -""" -Python Environment API for the ML-Agents Toolkit -The aim of this API is to expose Agents evolving in a simulation -to perform reinforcement learning on. -This API supports multi-agent scenarios and groups similar Agents (same -observations, actions spaces and behavior) together. These groups of Agents are -identified by their BehaviorName. -For performance reasons, the data of each group of agents is processed in a -batched manner. Agents are identified by a unique AgentId identifier that -allows tracking of Agents across simulation steps. Note that there is no -guarantee that the number or order of the Agents in the state will be -consistent across simulation steps. -A simulation steps corresponds to moving the simulation forward until at least -one agent in the simulation sends its observations to Python again. Since -Agents can request decisions at different frequencies, a simulation step does -not necessarily correspond to a fixed simulation time increment. -""" - -from abc import ABC, abstractmethod -from collections.abc import Mapping -from typing import ( - List, - NamedTuple, - Tuple, - Optional, - Dict, - Iterator, - Any, - Mapping as MappingType, -) -from enum import IntFlag, Enum -import numpy as np - -from mlagents_envs.exception import UnityActionException - -AgentId = int -GroupId = int -BehaviorName = str - - -class DecisionStep(NamedTuple): - """ - Contains the data a single Agent collected since the last - simulation step. - - obs is a list of numpy arrays observations collected by the agent. - - reward is a float. Corresponds to the rewards collected by the agent - since the last simulation step. - - agent_id is an int and an unique identifier for the corresponding Agent. - - action_mask is an optional list of one dimensional array of booleans. - Only available when using multi-discrete actions. - Each array corresponds to an action branch. Each array contains a mask - for each action of the branch. If true, the action is not available for - the agent during this simulation step. - """ - - obs: List[np.ndarray] - reward: float - agent_id: AgentId - action_mask: Optional[List[np.ndarray]] - group_id: int - group_reward: float - - -class DecisionSteps(Mapping): - """ - Contains the data a batch of similar Agents collected since the last - simulation step. Note that all Agents do not necessarily have new - information to send at each simulation step. Therefore, the ordering of - agents and the batch size of the DecisionSteps are not fixed across - simulation steps. - - obs is a list of numpy arrays observations collected by the batch of - agent. Each obs has one extra dimension compared to DecisionStep: the - first dimension of the array corresponds to the batch size of the batch. - - reward is a float vector of length batch size. Corresponds to the - rewards collected by each agent since the last simulation step. - - agent_id is an int vector of length batch size containing unique - identifier for the corresponding Agent. This is used to track Agents - across simulation steps. - - action_mask is an optional list of two dimensional array of booleans. - Only available when using multi-discrete actions. - Each array corresponds to an action branch. The first dimension of each - array is the batch size and the second contains a mask for each action of - the branch. If true, the action is not available for the agent during - this simulation step. - """ - - def __init__(self, obs, reward, agent_id, action_mask, group_id, group_reward): - self.obs: List[np.ndarray] = obs - self.reward: np.ndarray = reward - self.agent_id: np.ndarray = agent_id - self.action_mask: Optional[List[np.ndarray]] = action_mask - self.group_id: np.ndarray = group_id - self.group_reward: np.ndarray = group_reward - self._agent_id_to_index: Optional[Dict[AgentId, int]] = None - - @property - def agent_id_to_index(self) -> Dict[AgentId, int]: - """ - :returns: A Dict that maps agent_id to the index of those agents in - this DecisionSteps. - """ - if self._agent_id_to_index is None: - self._agent_id_to_index = {} - for a_idx, a_id in enumerate(self.agent_id): - self._agent_id_to_index[a_id] = a_idx - return self._agent_id_to_index - - def __len__(self) -> int: - return len(self.agent_id) - - def __getitem__(self, agent_id: AgentId) -> DecisionStep: - """ - returns the DecisionStep for a specific agent. - :param agent_id: The id of the agent - :returns: The DecisionStep - """ - if agent_id not in self.agent_id_to_index: - raise KeyError(f"agent_id {agent_id} is not present in the DecisionSteps") - agent_index = self._agent_id_to_index[agent_id] # type: ignore - agent_obs = [] - for batched_obs in self.obs: - agent_obs.append(batched_obs[agent_index]) - agent_mask = None - if self.action_mask is not None: - agent_mask = [] - for mask in self.action_mask: - agent_mask.append(mask[agent_index]) - group_id = self.group_id[agent_index] - return DecisionStep( - obs=agent_obs, - reward=self.reward[agent_index], - agent_id=agent_id, - action_mask=agent_mask, - group_id=group_id, - group_reward=self.group_reward[agent_index], - ) - - def __iter__(self) -> Iterator[Any]: - yield from self.agent_id - - @staticmethod - def empty(spec: "BehaviorSpec") -> "DecisionSteps": - """ - Returns an empty DecisionSteps. - :param spec: The BehaviorSpec for the DecisionSteps - """ - obs: List[np.ndarray] = [] - for sen_spec in spec.observation_specs: - obs += [np.zeros((0,) + sen_spec.shape, dtype=np.float32)] - return DecisionSteps( - obs=obs, - reward=np.zeros(0, dtype=np.float32), - agent_id=np.zeros(0, dtype=np.int32), - action_mask=None, - group_id=np.zeros(0, dtype=np.int32), - group_reward=np.zeros(0, dtype=np.float32), - ) - - -class TerminalStep(NamedTuple): - """ - Contains the data a single Agent collected when its episode ended. - - obs is a list of numpy arrays observations collected by the agent. - - reward is a float. Corresponds to the rewards collected by the agent - since the last simulation step. - - interrupted is a bool. Is true if the Agent was interrupted since the last - decision step. For example, if the Agent reached the maximum number of steps for - the episode. - - agent_id is an int and an unique identifier for the corresponding Agent. - """ - - obs: List[np.ndarray] - reward: float - interrupted: bool - agent_id: AgentId - group_id: GroupId - group_reward: float - - -class TerminalSteps(Mapping): - """ - Contains the data a batch of Agents collected when their episode - terminated. All Agents present in the TerminalSteps have ended their - episode. - - obs is a list of numpy arrays observations collected by the batch of - agent. Each obs has one extra dimension compared to DecisionStep: the - first dimension of the array corresponds to the batch size of the batch. - - reward is a float vector of length batch size. Corresponds to the - rewards collected by each agent since the last simulation step. - - interrupted is an array of booleans of length batch size. Is true if the - associated Agent was interrupted since the last decision step. For example, if the - Agent reached the maximum number of steps for the episode. - - agent_id is an int vector of length batch size containing unique - identifier for the corresponding Agent. This is used to track Agents - across simulation steps. - """ - - def __init__(self, obs, reward, interrupted, agent_id, group_id, group_reward): - self.obs: List[np.ndarray] = obs - self.reward: np.ndarray = reward - self.interrupted: np.ndarray = interrupted - self.agent_id: np.ndarray = agent_id - self.group_id: np.ndarray = group_id - self.group_reward: np.ndarray = group_reward - self._agent_id_to_index: Optional[Dict[AgentId, int]] = None - - @property - def agent_id_to_index(self) -> Dict[AgentId, int]: - """ - :returns: A Dict that maps agent_id to the index of those agents in - this TerminalSteps. - """ - if self._agent_id_to_index is None: - self._agent_id_to_index = {} - for a_idx, a_id in enumerate(self.agent_id): - self._agent_id_to_index[a_id] = a_idx - return self._agent_id_to_index - - def __len__(self) -> int: - return len(self.agent_id) - - def __getitem__(self, agent_id: AgentId) -> TerminalStep: - """ - returns the TerminalStep for a specific agent. - :param agent_id: The id of the agent - :returns: obs, reward, done, agent_id and optional action mask for a - specific agent - """ - if agent_id not in self.agent_id_to_index: - raise KeyError(f"agent_id {agent_id} is not present in the TerminalSteps") - agent_index = self._agent_id_to_index[agent_id] # type: ignore - agent_obs = [] - for batched_obs in self.obs: - agent_obs.append(batched_obs[agent_index]) - group_id = self.group_id[agent_index] - return TerminalStep( - obs=agent_obs, - reward=self.reward[agent_index], - interrupted=self.interrupted[agent_index], - agent_id=agent_id, - group_id=group_id, - group_reward=self.group_reward[agent_index], - ) - - def __iter__(self) -> Iterator[Any]: - yield from self.agent_id - - @staticmethod - def empty(spec: "BehaviorSpec") -> "TerminalSteps": - """ - Returns an empty TerminalSteps. - :param spec: The BehaviorSpec for the TerminalSteps - """ - obs: List[np.ndarray] = [] - for sen_spec in spec.observation_specs: - obs += [np.zeros((0,) + sen_spec.shape, dtype=np.float32)] - return TerminalSteps( - obs=obs, - reward=np.zeros(0, dtype=np.float32), - interrupted=np.zeros(0, dtype=np.bool), - agent_id=np.zeros(0, dtype=np.int32), - group_id=np.zeros(0, dtype=np.int32), - group_reward=np.zeros(0, dtype=np.float32), - ) - - -class _ActionTupleBase(ABC): - """ - An object whose fields correspond to action data of continuous and discrete - spaces. Dimensions are of (n_agents, continuous_size) and (n_agents, discrete_size), - respectively. Note, this also holds when continuous or discrete size is - zero. - """ - - def __init__( - self, - continuous: Optional[np.ndarray] = None, - discrete: Optional[np.ndarray] = None, - ): - self._continuous: Optional[np.ndarray] = None - self._discrete: Optional[np.ndarray] = None - if continuous is not None: - self.add_continuous(continuous) - if discrete is not None: - self.add_discrete(discrete) - - @property - def continuous(self) -> np.ndarray: - return self._continuous - - @property - def discrete(self) -> np.ndarray: - return self._discrete - - def add_continuous(self, continuous: np.ndarray) -> None: - if continuous.dtype != np.float32: - continuous = continuous.astype(np.float32, copy=False) - if self._discrete is None: - self._discrete = np.zeros( - (continuous.shape[0], 0), dtype=self.discrete_dtype - ) - self._continuous = continuous - - def add_discrete(self, discrete: np.ndarray) -> None: - if discrete.dtype != self.discrete_dtype: - discrete = discrete.astype(self.discrete_dtype, copy=False) - if self._continuous is None: - self._continuous = np.zeros((discrete.shape[0], 0), dtype=np.float32) - self._discrete = discrete - - @property - @abstractmethod - def discrete_dtype(self) -> np.dtype: - pass - - -class ActionTuple(_ActionTupleBase): - """ - An object whose fields correspond to actions of different types. - Continuous and discrete actions are numpy arrays of type float32 and - int32, respectively and are type checked on construction. - Dimensions are of (n_agents, continuous_size) and (n_agents, discrete_size), - respectively. Note, this also holds when continuous or discrete size is - zero. - """ - - @property - def discrete_dtype(self) -> np.dtype: - """ - The dtype of a discrete action. - """ - return np.int32 - - -class ActionSpec(NamedTuple): - """ - A NamedTuple containing utility functions and information about the action spaces - for a group of Agents under the same behavior. - - num_continuous_actions is an int corresponding to the number of floats which - constitute the action. - - discrete_branch_sizes is a Tuple of int where each int corresponds to - the number of discrete actions available to the agent on an independent action branch. - """ - - continuous_size: int - discrete_branches: Tuple[int, ...] - - def __eq__(self, other): - return ( - self.continuous_size == other.continuous_size - and self.discrete_branches == other.discrete_branches - ) - - def __str__(self): - return f"Continuous: {self.continuous_size}, Discrete: {self.discrete_branches}" - - # For backwards compatibility - def is_discrete(self) -> bool: - """ - Returns true if this Behavior uses discrete actions - """ - return self.discrete_size > 0 and self.continuous_size == 0 - - # For backwards compatibility - def is_continuous(self) -> bool: - """ - Returns true if this Behavior uses continuous actions - """ - return self.discrete_size == 0 and self.continuous_size > 0 - - @property - def discrete_size(self) -> int: - """ - Returns a an int corresponding to the number of discrete branches. - """ - return len(self.discrete_branches) - - def empty_action(self, n_agents: int) -> ActionTuple: - """ - Generates ActionTuple corresponding to an empty action (all zeros) - for a number of agents. - :param n_agents: The number of agents that will have actions generated - """ - _continuous = np.zeros((n_agents, self.continuous_size), dtype=np.float32) - _discrete = np.zeros((n_agents, self.discrete_size), dtype=np.int32) - return ActionTuple(continuous=_continuous, discrete=_discrete) - - def random_action(self, n_agents: int) -> ActionTuple: - """ - Generates ActionTuple corresponding to a random action (either discrete - or continuous) for a number of agents. - :param n_agents: The number of agents that will have actions generated - """ - _continuous = np.random.uniform( - low=-1.0, high=1.0, size=(n_agents, self.continuous_size) - ) - _discrete = np.zeros((n_agents, self.discrete_size), dtype=np.int32) - if self.discrete_size > 0: - _discrete = np.column_stack( - [ - np.random.randint( - 0, - self.discrete_branches[i], # type: ignore - size=(n_agents), - dtype=np.int32, - ) - for i in range(self.discrete_size) - ] - ) - return ActionTuple(continuous=_continuous, discrete=_discrete) - - def _validate_action( - self, actions: ActionTuple, n_agents: int, name: str - ) -> ActionTuple: - """ - Validates that action has the correct action dim - for the correct number of agents and ensures the type. - """ - _expected_shape = (n_agents, self.continuous_size) - if actions.continuous.shape != _expected_shape: - raise UnityActionException( - f"The behavior {name} needs a continuous input of dimension " - f"{_expected_shape} for (, ) but " - f"received input of dimension {actions.continuous.shape}" - ) - _expected_shape = (n_agents, self.discrete_size) - if actions.discrete.shape != _expected_shape: - raise UnityActionException( - f"The behavior {name} needs a discrete input of dimension " - f"{_expected_shape} for (, ) but " - f"received input of dimension {actions.discrete.shape}" - ) - return actions - - @staticmethod - def create_continuous(continuous_size: int) -> "ActionSpec": - """ - Creates an ActionSpec that is homogenously continuous - """ - return ActionSpec(continuous_size, ()) - - @staticmethod - def create_discrete(discrete_branches: Tuple[int]) -> "ActionSpec": - """ - Creates an ActionSpec that is homogenously discrete - """ - return ActionSpec(0, discrete_branches) - - -class DimensionProperty(IntFlag): - """ - The dimension property of a dimension of an observation. - """ - - UNSPECIFIED = 0 - """ - No properties specified. - """ - - NONE = 1 - """ - No Property of the observation in that dimension. Observation can be processed with - Fully connected networks. - """ - - TRANSLATIONAL_EQUIVARIANCE = 2 - """ - Means it is suitable to do a convolution in this dimension. - """ - - VARIABLE_SIZE = 4 - """ - Means that there can be a variable number of observations in this dimension. - The observations are unordered. - """ - - -class ObservationType(Enum): - """ - An Enum which defines the type of information carried in the observation - of the agent. - """ - - DEFAULT = 0 - """ - Observation information is generic. - """ - - GOAL_SIGNAL = 1 - """ - Observation contains goal information for current task. - """ - - -class ObservationSpec(NamedTuple): - """ - A NamedTuple containing information about the observation of Agents. - - shape is a Tuple of int : It corresponds to the shape of - an observation's dimensions. - - dimension_property is a Tuple of DimensionProperties flag, one flag for each - dimension. - - observation_type is an enum of ObservationType. - """ - - shape: Tuple[int, ...] - dimension_property: Tuple[DimensionProperty, ...] - observation_type: ObservationType - - # Optional name. For observations coming from com.unity.ml-agents, this - # will be the ISensor name. - name: str - - -class BehaviorSpec(NamedTuple): - """ - A NamedTuple containing information about the observation and action - spaces for a group of Agents under the same behavior. - - observation_specs is a List of ObservationSpec NamedTuple containing - information about the information of the Agent's observations such as their shapes. - The order of the ObservationSpec is the same as the order of the observations of an - agent. - - action_spec is an ActionSpec NamedTuple. - """ - - observation_specs: List[ObservationSpec] - action_spec: ActionSpec - - -class BehaviorMapping(Mapping): - def __init__(self, specs: Dict[BehaviorName, BehaviorSpec]): - self._dict = specs - - def __len__(self) -> int: - return len(self._dict) - - def __getitem__(self, behavior: BehaviorName) -> BehaviorSpec: - return self._dict[behavior] - - def __iter__(self) -> Iterator[Any]: - yield from self._dict - - -class BaseEnv(ABC): - @abstractmethod - def step(self) -> None: - """ - Signals the environment that it must move the simulation forward - by one step. - """ - - @abstractmethod - def reset(self) -> None: - """ - Signals the environment that it must reset the simulation. - """ - - @abstractmethod - def close(self) -> None: - """ - Signals the environment that it must close. - """ - - @property - @abstractmethod - def behavior_specs(self) -> MappingType[str, BehaviorSpec]: - """ - Returns a Mapping from behavior names to behavior specs. - Agents grouped under the same behavior name have the same action and - observation specs, and are expected to behave similarly in the - environment. - Note that new keys can be added to this mapping as new policies are instantiated. - """ - - @abstractmethod - def set_actions(self, behavior_name: BehaviorName, action: ActionTuple) -> None: - """ - Sets the action for all of the agents in the simulation for the next - step. The Actions must be in the same order as the order received in - the DecisionSteps. - :param behavior_name: The name of the behavior the agents are part of - :param action: ActionTuple tuple of continuous and/or discrete action. - Actions are np.arrays with dimensions (n_agents, continuous_size) and - (n_agents, discrete_size), respectively. - """ - - @abstractmethod - def set_action_for_agent( - self, behavior_name: BehaviorName, agent_id: AgentId, action: ActionTuple - ) -> None: - """ - Sets the action for one of the agents in the simulation for the next - step. - :param behavior_name: The name of the behavior the agent is part of - :param agent_id: The id of the agent the action is set for - :param action: ActionTuple tuple of continuous and/or discrete action - Actions are np.arrays with dimensions (1, continuous_size) and - (1, discrete_size), respectively. Note, this initial dimensions of 1 is because - this action is meant for a single agent. - """ - - @abstractmethod - def get_steps( - self, behavior_name: BehaviorName - ) -> Tuple[DecisionSteps, TerminalSteps]: - """ - Retrieves the steps of the agents that requested a step in the - simulation. - :param behavior_name: The name of the behavior the agents are part of - :return: A tuple containing : - - A DecisionSteps NamedTuple containing the observations, - the rewards, the agent ids and the action masks for the Agents - of the specified behavior. These Agents need an action this step. - - A TerminalSteps NamedTuple containing the observations, - rewards, agent ids and interrupted flags of the agents that had their - episode terminated last step. - """ diff --git a/3DOrangeTree/mlagents_envs/communicator.py b/3DOrangeTree/mlagents_envs/communicator.py deleted file mode 100644 index 2223f34..0000000 --- a/3DOrangeTree/mlagents_envs/communicator.py +++ /dev/null @@ -1,43 +0,0 @@ -from typing import Callable, Optional -from mlagents_envs.communicator_objects.unity_output_pb2 import UnityOutputProto -from mlagents_envs.communicator_objects.unity_input_pb2 import UnityInputProto - - -# Function to call while waiting for a connection timeout. -# This should raise an exception if it needs to break from waiting for the timeout. -PollCallback = Callable[[], None] - - -class Communicator: - def __init__(self, worker_id=0, base_port=5005): - """ - Python side of the communication. Must be used in pair with the right Unity Communicator equivalent. - - :int worker_id: Offset from base_port. Used for training multiple environments simultaneously. - :int base_port: Baseline port number to connect to Unity environment over. worker_id increments over this. - """ - - def initialize( - self, inputs: UnityInputProto, poll_callback: Optional[PollCallback] = None - ) -> UnityOutputProto: - """ - Used to exchange initialization parameters between Python and the Environment - :param inputs: The initialization input that will be sent to the environment. - :param poll_callback: Optional callback to be used while polling the connection. - :return: UnityOutput: The initialization output sent by Unity - """ - - def exchange( - self, inputs: UnityInputProto, poll_callback: Optional[PollCallback] = None - ) -> Optional[UnityOutputProto]: - """ - Used to send an input and receive an output from the Environment - :param inputs: The UnityInput that needs to be sent the Environment - :param poll_callback: Optional callback to be used while polling the connection. - :return: The UnityOutputs generated by the Environment - """ - - def close(self): - """ - Sends a shutdown signal to the unity environment, and closes the connection. - """ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__init__.py b/3DOrangeTree/mlagents_envs/communicator_objects/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/__init__.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index 0c3513d..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/__init__.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index ebe062d..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_action_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_action_pb2.cpython-36.pyc deleted file mode 100644 index ce3e796..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_action_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_action_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_action_pb2.cpython-37.pyc deleted file mode 100644 index 46b864e..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_action_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_info_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_info_pb2.cpython-36.pyc deleted file mode 100644 index c5dd40b..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_info_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_info_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_info_pb2.cpython-37.pyc deleted file mode 100644 index 98acd3b..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/agent_info_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/brain_parameters_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/brain_parameters_pb2.cpython-36.pyc deleted file mode 100644 index 890379a..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/brain_parameters_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/brain_parameters_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/brain_parameters_pb2.cpython-37.pyc deleted file mode 100644 index afe5124..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/brain_parameters_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/capabilities_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/capabilities_pb2.cpython-36.pyc deleted file mode 100644 index 4c2d608..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/capabilities_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/capabilities_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/capabilities_pb2.cpython-37.pyc deleted file mode 100644 index af77671..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/capabilities_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/command_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/command_pb2.cpython-36.pyc deleted file mode 100644 index 4b3158c..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/command_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/command_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/command_pb2.cpython-37.pyc deleted file mode 100644 index f9a257a..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/command_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/header_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/header_pb2.cpython-36.pyc deleted file mode 100644 index cfde38b..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/header_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/header_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/header_pb2.cpython-37.pyc deleted file mode 100644 index 1c2bb04..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/header_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/observation_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/observation_pb2.cpython-36.pyc deleted file mode 100644 index aaeb9ea..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/observation_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/observation_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/observation_pb2.cpython-37.pyc deleted file mode 100644 index 0412130..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/observation_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/space_type_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/space_type_pb2.cpython-36.pyc deleted file mode 100644 index e0da9bc..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/space_type_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/space_type_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/space_type_pb2.cpython-37.pyc deleted file mode 100644 index 9b2d033..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/space_type_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/training_analytics_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/training_analytics_pb2.cpython-36.pyc deleted file mode 100644 index 99667ba..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/training_analytics_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/training_analytics_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/training_analytics_pb2.cpython-37.pyc deleted file mode 100644 index c867d2a..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/training_analytics_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_input_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_input_pb2.cpython-36.pyc deleted file mode 100644 index 018e23b..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_input_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_input_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_input_pb2.cpython-37.pyc deleted file mode 100644 index 6efdf60..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_input_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_message_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_message_pb2.cpython-36.pyc deleted file mode 100644 index 5973ec6..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_message_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_message_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_message_pb2.cpython-37.pyc deleted file mode 100644 index 368ec41..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_message_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_output_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_output_pb2.cpython-36.pyc deleted file mode 100644 index 454819a..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_output_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_output_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_output_pb2.cpython-37.pyc deleted file mode 100644 index 325fb79..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_output_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_input_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_input_pb2.cpython-36.pyc deleted file mode 100644 index 4527793..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_input_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_input_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_input_pb2.cpython-37.pyc deleted file mode 100644 index ddad072..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_input_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_output_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_output_pb2.cpython-36.pyc deleted file mode 100644 index 961f7fe..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_output_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_output_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_output_pb2.cpython-37.pyc deleted file mode 100644 index e0a9870..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_initialization_output_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_input_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_input_pb2.cpython-36.pyc deleted file mode 100644 index edad3d6..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_input_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_input_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_input_pb2.cpython-37.pyc deleted file mode 100644 index e9f4c35..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_input_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_output_pb2.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_output_pb2.cpython-36.pyc deleted file mode 100644 index bd5c1da..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_output_pb2.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_output_pb2.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_output_pb2.cpython-37.pyc deleted file mode 100644 index 09d9613..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_rl_output_pb2.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_to_external_pb2_grpc.cpython-36.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_to_external_pb2_grpc.cpython-36.pyc deleted file mode 100644 index 1705425..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_to_external_pb2_grpc.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_to_external_pb2_grpc.cpython-37.pyc b/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_to_external_pb2_grpc.cpython-37.pyc deleted file mode 100644 index ab01acf..0000000 Binary files a/3DOrangeTree/mlagents_envs/communicator_objects/__pycache__/unity_to_external_pb2_grpc.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/agent_action_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/agent_action_pb2.py deleted file mode 100644 index 7c57d8b..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/agent_action_pb2.py +++ /dev/null @@ -1,92 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/agent_action.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/agent_action.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n5mlagents_envs/communicator_objects/agent_action.proto\x12\x14\x63ommunicator_objects\"\x8c\x01\n\x10\x41gentActionProto\x12!\n\x19vector_actions_deprecated\x18\x01 \x03(\x02\x12\r\n\x05value\x18\x04 \x01(\x02\x12\x1a\n\x12\x63ontinuous_actions\x18\x06 \x03(\x02\x12\x18\n\x10\x64iscrete_actions\x18\x07 \x03(\x05J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04J\x04\x08\x05\x10\x06\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - - - - -_AGENTACTIONPROTO = _descriptor.Descriptor( - name='AgentActionProto', - full_name='communicator_objects.AgentActionProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='vector_actions_deprecated', full_name='communicator_objects.AgentActionProto.vector_actions_deprecated', index=0, - number=1, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='communicator_objects.AgentActionProto.value', index=1, - number=4, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='continuous_actions', full_name='communicator_objects.AgentActionProto.continuous_actions', index=2, - number=6, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='discrete_actions', full_name='communicator_objects.AgentActionProto.discrete_actions', index=3, - number=7, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=80, - serialized_end=220, -) - -DESCRIPTOR.message_types_by_name['AgentActionProto'] = _AGENTACTIONPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -AgentActionProto = _reflection.GeneratedProtocolMessageType('AgentActionProto', (_message.Message,), dict( - DESCRIPTOR = _AGENTACTIONPROTO, - __module__ = 'mlagents_envs.communicator_objects.agent_action_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.AgentActionProto) - )) -_sym_db.RegisterMessage(AgentActionProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/agent_action_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/agent_action_pb2.pyi deleted file mode 100644 index b3ddc28..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/agent_action_pb2.pyi +++ /dev/null @@ -1,52 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.internal.containers import ( - RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - Iterable as typing___Iterable, - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class AgentActionProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - vector_actions_deprecated = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] - value = ... # type: builtin___float - continuous_actions = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] - discrete_actions = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] - - def __init__(self, - *, - vector_actions_deprecated : typing___Optional[typing___Iterable[builtin___float]] = None, - value : typing___Optional[builtin___float] = None, - continuous_actions : typing___Optional[typing___Iterable[builtin___float]] = None, - discrete_actions : typing___Optional[typing___Iterable[builtin___int]] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> AgentActionProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"continuous_actions",u"discrete_actions",u"value",u"vector_actions_deprecated"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"continuous_actions",b"continuous_actions",u"discrete_actions",b"discrete_actions",u"value",b"value",u"vector_actions_deprecated",b"vector_actions_deprecated"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_action_pair_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_action_pair_pb2.py deleted file mode 100644 index 667b91a..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_action_pair_pb2.py +++ /dev/null @@ -1,83 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/agent_info_action_pair.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import agent_info_pb2 as mlagents__envs_dot_communicator__objects_dot_agent__info__pb2 -from mlagents_envs.communicator_objects import agent_action_pb2 as mlagents__envs_dot_communicator__objects_dot_agent__action__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/agent_info_action_pair.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n?mlagents_envs/communicator_objects/agent_info_action_pair.proto\x12\x14\x63ommunicator_objects\x1a\x33mlagents_envs/communicator_objects/agent_info.proto\x1a\x35mlagents_envs/communicator_objects/agent_action.proto\"\x91\x01\n\x18\x41gentInfoActionPairProto\x12\x38\n\nagent_info\x18\x01 \x01(\x0b\x32$.communicator_objects.AgentInfoProto\x12;\n\x0b\x61\x63tion_info\x18\x02 \x01(\x0b\x32&.communicator_objects.AgentActionProtoB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_agent__info__pb2.DESCRIPTOR,mlagents__envs_dot_communicator__objects_dot_agent__action__pb2.DESCRIPTOR,]) - - - - -_AGENTINFOACTIONPAIRPROTO = _descriptor.Descriptor( - name='AgentInfoActionPairProto', - full_name='communicator_objects.AgentInfoActionPairProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='agent_info', full_name='communicator_objects.AgentInfoActionPairProto.agent_info', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='action_info', full_name='communicator_objects.AgentInfoActionPairProto.action_info', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=198, - serialized_end=343, -) - -_AGENTINFOACTIONPAIRPROTO.fields_by_name['agent_info'].message_type = mlagents__envs_dot_communicator__objects_dot_agent__info__pb2._AGENTINFOPROTO -_AGENTINFOACTIONPAIRPROTO.fields_by_name['action_info'].message_type = mlagents__envs_dot_communicator__objects_dot_agent__action__pb2._AGENTACTIONPROTO -DESCRIPTOR.message_types_by_name['AgentInfoActionPairProto'] = _AGENTINFOACTIONPAIRPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -AgentInfoActionPairProto = _reflection.GeneratedProtocolMessageType('AgentInfoActionPairProto', (_message.Message,), dict( - DESCRIPTOR = _AGENTINFOACTIONPAIRPROTO, - __module__ = 'mlagents_envs.communicator_objects.agent_info_action_pair_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.AgentInfoActionPairProto) - )) -_sym_db.RegisterMessage(AgentInfoActionPairProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_action_pair_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_action_pair_pb2.pyi deleted file mode 100644 index e3711aa..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_action_pair_pb2.pyi +++ /dev/null @@ -1,57 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.agent_action_pb2 import ( - AgentActionProto as mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto, -) - -from mlagents_envs.communicator_objects.agent_info_pb2 import ( - AgentInfoProto as mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto, -) - -from typing import ( - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class AgentInfoActionPairProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - - @property - def agent_info(self) -> mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto: ... - - @property - def action_info(self) -> mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto: ... - - def __init__(self, - *, - agent_info : typing___Optional[mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto] = None, - action_info : typing___Optional[mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> AgentInfoActionPairProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"action_info",u"agent_info"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"action_info",u"agent_info"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"action_info",b"action_info",u"agent_info",b"agent_info"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"action_info",b"action_info",u"agent_info",b"agent_info"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_pb2.py deleted file mode 100644 index 57bb77a..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_pb2.py +++ /dev/null @@ -1,123 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/agent_info.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import observation_pb2 as mlagents__envs_dot_communicator__objects_dot_observation__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/agent_info.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n3mlagents_envs/communicator_objects/agent_info.proto\x12\x14\x63ommunicator_objects\x1a\x34mlagents_envs/communicator_objects/observation.proto\"\xf9\x01\n\x0e\x41gentInfoProto\x12\x0e\n\x06reward\x18\x07 \x01(\x02\x12\x0c\n\x04\x64one\x18\x08 \x01(\x08\x12\x18\n\x10max_step_reached\x18\t \x01(\x08\x12\n\n\x02id\x18\n \x01(\x05\x12\x13\n\x0b\x61\x63tion_mask\x18\x0b \x03(\x08\x12<\n\x0cobservations\x18\r \x03(\x0b\x32&.communicator_objects.ObservationProto\x12\x10\n\x08group_id\x18\x0e \x01(\x05\x12\x14\n\x0cgroup_reward\x18\x0f \x01(\x02J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x0c\x10\rB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_observation__pb2.DESCRIPTOR,]) - - - - -_AGENTINFOPROTO = _descriptor.Descriptor( - name='AgentInfoProto', - full_name='communicator_objects.AgentInfoProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='reward', full_name='communicator_objects.AgentInfoProto.reward', index=0, - number=7, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='done', full_name='communicator_objects.AgentInfoProto.done', index=1, - number=8, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='max_step_reached', full_name='communicator_objects.AgentInfoProto.max_step_reached', index=2, - number=9, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='id', full_name='communicator_objects.AgentInfoProto.id', index=3, - number=10, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='action_mask', full_name='communicator_objects.AgentInfoProto.action_mask', index=4, - number=11, type=8, cpp_type=7, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='observations', full_name='communicator_objects.AgentInfoProto.observations', index=5, - number=13, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='group_id', full_name='communicator_objects.AgentInfoProto.group_id', index=6, - number=14, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='group_reward', full_name='communicator_objects.AgentInfoProto.group_reward', index=7, - number=15, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=132, - serialized_end=381, -) - -_AGENTINFOPROTO.fields_by_name['observations'].message_type = mlagents__envs_dot_communicator__objects_dot_observation__pb2._OBSERVATIONPROTO -DESCRIPTOR.message_types_by_name['AgentInfoProto'] = _AGENTINFOPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -AgentInfoProto = _reflection.GeneratedProtocolMessageType('AgentInfoProto', (_message.Message,), dict( - DESCRIPTOR = _AGENTINFOPROTO, - __module__ = 'mlagents_envs.communicator_objects.agent_info_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.AgentInfoProto) - )) -_sym_db.RegisterMessage(AgentInfoProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_pb2.pyi deleted file mode 100644 index 821d242..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/agent_info_pb2.pyi +++ /dev/null @@ -1,67 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.internal.containers import ( - RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, - RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.observation_pb2 import ( - ObservationProto as mlagents_envs___communicator_objects___observation_pb2___ObservationProto, -) - -from typing import ( - Iterable as typing___Iterable, - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class AgentInfoProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - reward = ... # type: builtin___float - done = ... # type: builtin___bool - max_step_reached = ... # type: builtin___bool - id = ... # type: builtin___int - action_mask = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bool] - group_id = ... # type: builtin___int - group_reward = ... # type: builtin___float - - @property - def observations(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents_envs___communicator_objects___observation_pb2___ObservationProto]: ... - - def __init__(self, - *, - reward : typing___Optional[builtin___float] = None, - done : typing___Optional[builtin___bool] = None, - max_step_reached : typing___Optional[builtin___bool] = None, - id : typing___Optional[builtin___int] = None, - action_mask : typing___Optional[typing___Iterable[builtin___bool]] = None, - observations : typing___Optional[typing___Iterable[mlagents_envs___communicator_objects___observation_pb2___ObservationProto]] = None, - group_id : typing___Optional[builtin___int] = None, - group_reward : typing___Optional[builtin___float] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> AgentInfoProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",u"done",u"group_id",u"group_reward",u"id",u"max_step_reached",u"observations",u"reward"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"action_mask",b"action_mask",u"done",b"done",u"group_id",b"group_id",u"group_reward",b"group_reward",u"id",b"id",u"max_step_reached",b"max_step_reached",u"observations",b"observations",u"reward",b"reward"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/brain_parameters_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/brain_parameters_pb2.py deleted file mode 100644 index ccd9140..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/brain_parameters_pb2.py +++ /dev/null @@ -1,170 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/brain_parameters.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import space_type_pb2 as mlagents__envs_dot_communicator__objects_dot_space__type__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/brain_parameters.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n9mlagents_envs/communicator_objects/brain_parameters.proto\x12\x14\x63ommunicator_objects\x1a\x33mlagents_envs/communicator_objects/space_type.proto\"\x8b\x01\n\x0f\x41\x63tionSpecProto\x12\x1e\n\x16num_continuous_actions\x18\x01 \x01(\x05\x12\x1c\n\x14num_discrete_actions\x18\x02 \x01(\x05\x12\x1d\n\x15\x64iscrete_branch_sizes\x18\x03 \x03(\x05\x12\x1b\n\x13\x61\x63tion_descriptions\x18\x04 \x03(\t\"\xb6\x02\n\x14\x42rainParametersProto\x12%\n\x1dvector_action_size_deprecated\x18\x03 \x03(\x05\x12-\n%vector_action_descriptions_deprecated\x18\x05 \x03(\t\x12Q\n#vector_action_space_type_deprecated\x18\x06 \x01(\x0e\x32$.communicator_objects.SpaceTypeProto\x12\x12\n\nbrain_name\x18\x07 \x01(\t\x12\x13\n\x0bis_training\x18\x08 \x01(\x08\x12:\n\x0b\x61\x63tion_spec\x18\t \x01(\x0b\x32%.communicator_objects.ActionSpecProtoJ\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_space__type__pb2.DESCRIPTOR,]) - - - - -_ACTIONSPECPROTO = _descriptor.Descriptor( - name='ActionSpecProto', - full_name='communicator_objects.ActionSpecProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='num_continuous_actions', full_name='communicator_objects.ActionSpecProto.num_continuous_actions', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_discrete_actions', full_name='communicator_objects.ActionSpecProto.num_discrete_actions', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='discrete_branch_sizes', full_name='communicator_objects.ActionSpecProto.discrete_branch_sizes', index=2, - number=3, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='action_descriptions', full_name='communicator_objects.ActionSpecProto.action_descriptions', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=137, - serialized_end=276, -) - - -_BRAINPARAMETERSPROTO = _descriptor.Descriptor( - name='BrainParametersProto', - full_name='communicator_objects.BrainParametersProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='vector_action_size_deprecated', full_name='communicator_objects.BrainParametersProto.vector_action_size_deprecated', index=0, - number=3, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='vector_action_descriptions_deprecated', full_name='communicator_objects.BrainParametersProto.vector_action_descriptions_deprecated', index=1, - number=5, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='vector_action_space_type_deprecated', full_name='communicator_objects.BrainParametersProto.vector_action_space_type_deprecated', index=2, - number=6, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='brain_name', full_name='communicator_objects.BrainParametersProto.brain_name', index=3, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='is_training', full_name='communicator_objects.BrainParametersProto.is_training', index=4, - number=8, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='action_spec', full_name='communicator_objects.BrainParametersProto.action_spec', index=5, - number=9, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=279, - serialized_end=589, -) - -_BRAINPARAMETERSPROTO.fields_by_name['vector_action_space_type_deprecated'].enum_type = mlagents__envs_dot_communicator__objects_dot_space__type__pb2._SPACETYPEPROTO -_BRAINPARAMETERSPROTO.fields_by_name['action_spec'].message_type = _ACTIONSPECPROTO -DESCRIPTOR.message_types_by_name['ActionSpecProto'] = _ACTIONSPECPROTO -DESCRIPTOR.message_types_by_name['BrainParametersProto'] = _BRAINPARAMETERSPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -ActionSpecProto = _reflection.GeneratedProtocolMessageType('ActionSpecProto', (_message.Message,), dict( - DESCRIPTOR = _ACTIONSPECPROTO, - __module__ = 'mlagents_envs.communicator_objects.brain_parameters_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.ActionSpecProto) - )) -_sym_db.RegisterMessage(ActionSpecProto) - -BrainParametersProto = _reflection.GeneratedProtocolMessageType('BrainParametersProto', (_message.Message,), dict( - DESCRIPTOR = _BRAINPARAMETERSPROTO, - __module__ = 'mlagents_envs.communicator_objects.brain_parameters_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.BrainParametersProto) - )) -_sym_db.RegisterMessage(BrainParametersProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/brain_parameters_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/brain_parameters_pb2.pyi deleted file mode 100644 index 19611e1..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/brain_parameters_pb2.pyi +++ /dev/null @@ -1,88 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.internal.containers import ( - RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.space_type_pb2 import ( - SpaceTypeProto as mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProto, -) - -from typing import ( - Iterable as typing___Iterable, - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class ActionSpecProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - num_continuous_actions = ... # type: builtin___int - num_discrete_actions = ... # type: builtin___int - discrete_branch_sizes = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] - action_descriptions = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] - - def __init__(self, - *, - num_continuous_actions : typing___Optional[builtin___int] = None, - num_discrete_actions : typing___Optional[builtin___int] = None, - discrete_branch_sizes : typing___Optional[typing___Iterable[builtin___int]] = None, - action_descriptions : typing___Optional[typing___Iterable[typing___Text]] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> ActionSpecProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"action_descriptions",u"discrete_branch_sizes",u"num_continuous_actions",u"num_discrete_actions"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"action_descriptions",b"action_descriptions",u"discrete_branch_sizes",b"discrete_branch_sizes",u"num_continuous_actions",b"num_continuous_actions",u"num_discrete_actions",b"num_discrete_actions"]) -> None: ... - -class BrainParametersProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - vector_action_size_deprecated = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] - vector_action_descriptions_deprecated = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] - vector_action_space_type_deprecated = ... # type: mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProto - brain_name = ... # type: typing___Text - is_training = ... # type: builtin___bool - - @property - def action_spec(self) -> ActionSpecProto: ... - - def __init__(self, - *, - vector_action_size_deprecated : typing___Optional[typing___Iterable[builtin___int]] = None, - vector_action_descriptions_deprecated : typing___Optional[typing___Iterable[typing___Text]] = None, - vector_action_space_type_deprecated : typing___Optional[mlagents_envs___communicator_objects___space_type_pb2___SpaceTypeProto] = None, - brain_name : typing___Optional[typing___Text] = None, - is_training : typing___Optional[builtin___bool] = None, - action_spec : typing___Optional[ActionSpecProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> BrainParametersProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"action_spec"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"action_spec",u"brain_name",u"is_training",u"vector_action_descriptions_deprecated",u"vector_action_size_deprecated",u"vector_action_space_type_deprecated"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"action_spec",b"action_spec"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"action_spec",b"action_spec",u"brain_name",b"brain_name",u"is_training",b"is_training",u"vector_action_descriptions_deprecated",b"vector_action_descriptions_deprecated",u"vector_action_size_deprecated",b"vector_action_size_deprecated",u"vector_action_space_type_deprecated",b"vector_action_space_type_deprecated"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/capabilities_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/capabilities_pb2.py deleted file mode 100644 index 35b8fbd..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/capabilities_pb2.py +++ /dev/null @@ -1,113 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/capabilities.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/capabilities.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n5mlagents_envs/communicator_objects/capabilities.proto\x12\x14\x63ommunicator_objects\"\xec\x01\n\x18UnityRLCapabilitiesProto\x12\x1a\n\x12\x62\x61seRLCapabilities\x18\x01 \x01(\x08\x12#\n\x1b\x63oncatenatedPngObservations\x18\x02 \x01(\x08\x12 \n\x18\x63ompressedChannelMapping\x18\x03 \x01(\x08\x12\x15\n\rhybridActions\x18\x04 \x01(\x08\x12\x19\n\x11trainingAnalytics\x18\x05 \x01(\x08\x12!\n\x19variableLengthObservation\x18\x06 \x01(\x08\x12\x18\n\x10multiAgentGroups\x18\x07 \x01(\x08\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - - - - -_UNITYRLCAPABILITIESPROTO = _descriptor.Descriptor( - name='UnityRLCapabilitiesProto', - full_name='communicator_objects.UnityRLCapabilitiesProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='baseRLCapabilities', full_name='communicator_objects.UnityRLCapabilitiesProto.baseRLCapabilities', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='concatenatedPngObservations', full_name='communicator_objects.UnityRLCapabilitiesProto.concatenatedPngObservations', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='compressedChannelMapping', full_name='communicator_objects.UnityRLCapabilitiesProto.compressedChannelMapping', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='hybridActions', full_name='communicator_objects.UnityRLCapabilitiesProto.hybridActions', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='trainingAnalytics', full_name='communicator_objects.UnityRLCapabilitiesProto.trainingAnalytics', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='variableLengthObservation', full_name='communicator_objects.UnityRLCapabilitiesProto.variableLengthObservation', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='multiAgentGroups', full_name='communicator_objects.UnityRLCapabilitiesProto.multiAgentGroups', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=80, - serialized_end=316, -) - -DESCRIPTOR.message_types_by_name['UnityRLCapabilitiesProto'] = _UNITYRLCAPABILITIESPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityRLCapabilitiesProto = _reflection.GeneratedProtocolMessageType('UnityRLCapabilitiesProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYRLCAPABILITIESPROTO, - __module__ = 'mlagents_envs.communicator_objects.capabilities_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLCapabilitiesProto) - )) -_sym_db.RegisterMessage(UnityRLCapabilitiesProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/capabilities_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/capabilities_pb2.pyi deleted file mode 100644 index 1c6a1f7..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/capabilities_pb2.pyi +++ /dev/null @@ -1,53 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityRLCapabilitiesProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - baseRLCapabilities = ... # type: builtin___bool - concatenatedPngObservations = ... # type: builtin___bool - compressedChannelMapping = ... # type: builtin___bool - hybridActions = ... # type: builtin___bool - trainingAnalytics = ... # type: builtin___bool - variableLengthObservation = ... # type: builtin___bool - multiAgentGroups = ... # type: builtin___bool - - def __init__(self, - *, - baseRLCapabilities : typing___Optional[builtin___bool] = None, - concatenatedPngObservations : typing___Optional[builtin___bool] = None, - compressedChannelMapping : typing___Optional[builtin___bool] = None, - hybridActions : typing___Optional[builtin___bool] = None, - trainingAnalytics : typing___Optional[builtin___bool] = None, - variableLengthObservation : typing___Optional[builtin___bool] = None, - multiAgentGroups : typing___Optional[builtin___bool] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLCapabilitiesProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"baseRLCapabilities",u"compressedChannelMapping",u"concatenatedPngObservations",u"hybridActions",u"multiAgentGroups",u"trainingAnalytics",u"variableLengthObservation"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"baseRLCapabilities",b"baseRLCapabilities",u"compressedChannelMapping",b"compressedChannelMapping",u"concatenatedPngObservations",b"concatenatedPngObservations",u"hybridActions",b"hybridActions",u"multiAgentGroups",b"multiAgentGroups",u"trainingAnalytics",b"trainingAnalytics",u"variableLengthObservation",b"variableLengthObservation"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/command_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/command_pb2.py deleted file mode 100644 index 397f310..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/command_pb2.py +++ /dev/null @@ -1,64 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/command.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/command.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n0mlagents_envs/communicator_objects/command.proto\x12\x14\x63ommunicator_objects*-\n\x0c\x43ommandProto\x12\x08\n\x04STEP\x10\x00\x12\t\n\x05RESET\x10\x01\x12\x08\n\x04QUIT\x10\x02\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - -_COMMANDPROTO = _descriptor.EnumDescriptor( - name='CommandProto', - full_name='communicator_objects.CommandProto', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='STEP', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RESET', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='QUIT', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=74, - serialized_end=119, -) -_sym_db.RegisterEnumDescriptor(_COMMANDPROTO) - -CommandProto = enum_type_wrapper.EnumTypeWrapper(_COMMANDPROTO) -STEP = 0 -RESET = 1 -QUIT = 2 - - -DESCRIPTOR.enum_types_by_name['CommandProto'] = _COMMANDPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/command_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/command_pb2.pyi deleted file mode 100644 index 4378a8a..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/command_pb2.pyi +++ /dev/null @@ -1,39 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - EnumDescriptor as google___protobuf___descriptor___EnumDescriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - List as typing___List, - Tuple as typing___Tuple, - cast as typing___cast, -) - - -builtin___int = int -builtin___str = str - - -class CommandProto(builtin___int): - DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... - @classmethod - def Name(cls, number: builtin___int) -> builtin___str: ... - @classmethod - def Value(cls, name: builtin___str) -> 'CommandProto': ... - @classmethod - def keys(cls) -> typing___List[builtin___str]: ... - @classmethod - def values(cls) -> typing___List['CommandProto']: ... - @classmethod - def items(cls) -> typing___List[typing___Tuple[builtin___str, 'CommandProto']]: ... - STEP = typing___cast('CommandProto', 0) - RESET = typing___cast('CommandProto', 1) - QUIT = typing___cast('CommandProto', 2) -STEP = typing___cast('CommandProto', 0) -RESET = typing___cast('CommandProto', 1) -QUIT = typing___cast('CommandProto', 2) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/custom_reset_parameters_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/custom_reset_parameters_pb2.py deleted file mode 100644 index f4300d1..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/custom_reset_parameters_pb2.py +++ /dev/null @@ -1,64 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/custom_reset_parameters.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/custom_reset_parameters.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n@mlagents_envs/communicator_objects/custom_reset_parameters.proto\x12\x14\x63ommunicator_objects\"\x1c\n\x1a\x43ustomResetParametersProtoB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - - - - -_CUSTOMRESETPARAMETERSPROTO = _descriptor.Descriptor( - name='CustomResetParametersProto', - full_name='communicator_objects.CustomResetParametersProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=90, - serialized_end=118, -) - -DESCRIPTOR.message_types_by_name['CustomResetParametersProto'] = _CUSTOMRESETPARAMETERSPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -CustomResetParametersProto = _reflection.GeneratedProtocolMessageType('CustomResetParametersProto', (_message.Message,), dict( - DESCRIPTOR = _CUSTOMRESETPARAMETERSPROTO, - __module__ = 'mlagents_envs.communicator_objects.custom_reset_parameters_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.CustomResetParametersProto) - )) -_sym_db.RegisterMessage(CustomResetParametersProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/custom_reset_parameters_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/custom_reset_parameters_pb2.pyi deleted file mode 100644 index 8127e97..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/custom_reset_parameters_pb2.pyi +++ /dev/null @@ -1,23 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - - -builtin___bytes = bytes - - -class CustomResetParametersProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - - def __init__(self, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> CustomResetParametersProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/demonstration_meta_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/demonstration_meta_pb2.py deleted file mode 100644 index 3b86fee..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/demonstration_meta_pb2.py +++ /dev/null @@ -1,99 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/demonstration_meta.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/demonstration_meta.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n;mlagents_envs/communicator_objects/demonstration_meta.proto\x12\x14\x63ommunicator_objects\"\x8d\x01\n\x16\x44\x65monstrationMetaProto\x12\x13\n\x0b\x61pi_version\x18\x01 \x01(\x05\x12\x1a\n\x12\x64\x65monstration_name\x18\x02 \x01(\t\x12\x14\n\x0cnumber_steps\x18\x03 \x01(\x05\x12\x17\n\x0fnumber_episodes\x18\x04 \x01(\x05\x12\x13\n\x0bmean_reward\x18\x05 \x01(\x02\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - - - - -_DEMONSTRATIONMETAPROTO = _descriptor.Descriptor( - name='DemonstrationMetaProto', - full_name='communicator_objects.DemonstrationMetaProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='api_version', full_name='communicator_objects.DemonstrationMetaProto.api_version', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='demonstration_name', full_name='communicator_objects.DemonstrationMetaProto.demonstration_name', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='number_steps', full_name='communicator_objects.DemonstrationMetaProto.number_steps', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='number_episodes', full_name='communicator_objects.DemonstrationMetaProto.number_episodes', index=3, - number=4, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mean_reward', full_name='communicator_objects.DemonstrationMetaProto.mean_reward', index=4, - number=5, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=86, - serialized_end=227, -) - -DESCRIPTOR.message_types_by_name['DemonstrationMetaProto'] = _DEMONSTRATIONMETAPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -DemonstrationMetaProto = _reflection.GeneratedProtocolMessageType('DemonstrationMetaProto', (_message.Message,), dict( - DESCRIPTOR = _DEMONSTRATIONMETAPROTO, - __module__ = 'mlagents_envs.communicator_objects.demonstration_meta_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.DemonstrationMetaProto) - )) -_sym_db.RegisterMessage(DemonstrationMetaProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/demonstration_meta_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/demonstration_meta_pb2.pyi deleted file mode 100644 index ba3c306..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/demonstration_meta_pb2.pyi +++ /dev/null @@ -1,50 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class DemonstrationMetaProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - api_version = ... # type: builtin___int - demonstration_name = ... # type: typing___Text - number_steps = ... # type: builtin___int - number_episodes = ... # type: builtin___int - mean_reward = ... # type: builtin___float - - def __init__(self, - *, - api_version : typing___Optional[builtin___int] = None, - demonstration_name : typing___Optional[typing___Text] = None, - number_steps : typing___Optional[builtin___int] = None, - number_episodes : typing___Optional[builtin___int] = None, - mean_reward : typing___Optional[builtin___float] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> DemonstrationMetaProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"api_version",u"demonstration_name",u"mean_reward",u"number_episodes",u"number_steps"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"api_version",b"api_version",u"demonstration_name",b"demonstration_name",u"mean_reward",b"mean_reward",u"number_episodes",b"number_episodes",u"number_steps",b"number_steps"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/engine_configuration_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/engine_configuration_pb2.py deleted file mode 100644 index 9f4656c..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/engine_configuration_pb2.py +++ /dev/null @@ -1,106 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/engine_configuration.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/engine_configuration.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n=mlagents_envs/communicator_objects/engine_configuration.proto\x12\x14\x63ommunicator_objects\"\x95\x01\n\x18\x45ngineConfigurationProto\x12\r\n\x05width\x18\x01 \x01(\x05\x12\x0e\n\x06height\x18\x02 \x01(\x05\x12\x15\n\rquality_level\x18\x03 \x01(\x05\x12\x12\n\ntime_scale\x18\x04 \x01(\x02\x12\x19\n\x11target_frame_rate\x18\x05 \x01(\x05\x12\x14\n\x0cshow_monitor\x18\x06 \x01(\x08\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - - - - -_ENGINECONFIGURATIONPROTO = _descriptor.Descriptor( - name='EngineConfigurationProto', - full_name='communicator_objects.EngineConfigurationProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='width', full_name='communicator_objects.EngineConfigurationProto.width', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='height', full_name='communicator_objects.EngineConfigurationProto.height', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='quality_level', full_name='communicator_objects.EngineConfigurationProto.quality_level', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='time_scale', full_name='communicator_objects.EngineConfigurationProto.time_scale', index=3, - number=4, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='target_frame_rate', full_name='communicator_objects.EngineConfigurationProto.target_frame_rate', index=4, - number=5, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='show_monitor', full_name='communicator_objects.EngineConfigurationProto.show_monitor', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=88, - serialized_end=237, -) - -DESCRIPTOR.message_types_by_name['EngineConfigurationProto'] = _ENGINECONFIGURATIONPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -EngineConfigurationProto = _reflection.GeneratedProtocolMessageType('EngineConfigurationProto', (_message.Message,), dict( - DESCRIPTOR = _ENGINECONFIGURATIONPROTO, - __module__ = 'mlagents_envs.communicator_objects.engine_configuration_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.EngineConfigurationProto) - )) -_sym_db.RegisterMessage(EngineConfigurationProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/engine_configuration_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/engine_configuration_pb2.pyi deleted file mode 100644 index d4eef07..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/engine_configuration_pb2.pyi +++ /dev/null @@ -1,51 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class EngineConfigurationProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - width = ... # type: builtin___int - height = ... # type: builtin___int - quality_level = ... # type: builtin___int - time_scale = ... # type: builtin___float - target_frame_rate = ... # type: builtin___int - show_monitor = ... # type: builtin___bool - - def __init__(self, - *, - width : typing___Optional[builtin___int] = None, - height : typing___Optional[builtin___int] = None, - quality_level : typing___Optional[builtin___int] = None, - time_scale : typing___Optional[builtin___float] = None, - target_frame_rate : typing___Optional[builtin___int] = None, - show_monitor : typing___Optional[builtin___bool] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> EngineConfigurationProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"height",u"quality_level",u"show_monitor",u"target_frame_rate",u"time_scale",u"width"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"height",b"height",u"quality_level",b"quality_level",u"show_monitor",b"show_monitor",u"target_frame_rate",b"target_frame_rate",u"time_scale",b"time_scale",u"width",b"width"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/header_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/header_pb2.py deleted file mode 100644 index 5aa8eb8..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/header_pb2.py +++ /dev/null @@ -1,78 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/header.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/header.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n/mlagents_envs/communicator_objects/header.proto\x12\x14\x63ommunicator_objects\".\n\x0bHeaderProto\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\tB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - - - - -_HEADERPROTO = _descriptor.Descriptor( - name='HeaderProto', - full_name='communicator_objects.HeaderProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='status', full_name='communicator_objects.HeaderProto.status', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='message', full_name='communicator_objects.HeaderProto.message', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=73, - serialized_end=119, -) - -DESCRIPTOR.message_types_by_name['HeaderProto'] = _HEADERPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -HeaderProto = _reflection.GeneratedProtocolMessageType('HeaderProto', (_message.Message,), dict( - DESCRIPTOR = _HEADERPROTO, - __module__ = 'mlagents_envs.communicator_objects.header_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.HeaderProto) - )) -_sym_db.RegisterMessage(HeaderProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/header_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/header_pb2.pyi deleted file mode 100644 index ab7fdd6..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/header_pb2.pyi +++ /dev/null @@ -1,44 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class HeaderProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - status = ... # type: builtin___int - message = ... # type: typing___Text - - def __init__(self, - *, - status : typing___Optional[builtin___int] = None, - message : typing___Optional[typing___Text] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> HeaderProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"message",u"status"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"message",b"message",u"status",b"status"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/observation_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/observation_pb2.py deleted file mode 100644 index 838ca1d..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/observation_pb2.py +++ /dev/null @@ -1,224 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/observation.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/observation.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n4mlagents_envs/communicator_objects/observation.proto\x12\x14\x63ommunicator_objects\"\x8f\x03\n\x10ObservationProto\x12\r\n\x05shape\x18\x01 \x03(\x05\x12\x44\n\x10\x63ompression_type\x18\x02 \x01(\x0e\x32*.communicator_objects.CompressionTypeProto\x12\x19\n\x0f\x63ompressed_data\x18\x03 \x01(\x0cH\x00\x12\x46\n\nfloat_data\x18\x04 \x01(\x0b\x32\x30.communicator_objects.ObservationProto.FloatDataH\x00\x12\"\n\x1a\x63ompressed_channel_mapping\x18\x05 \x03(\x05\x12\x1c\n\x14\x64imension_properties\x18\x06 \x03(\x05\x12\x44\n\x10observation_type\x18\x07 \x01(\x0e\x32*.communicator_objects.ObservationTypeProto\x12\x0c\n\x04name\x18\x08 \x01(\t\x1a\x19\n\tFloatData\x12\x0c\n\x04\x64\x61ta\x18\x01 \x03(\x02\x42\x12\n\x10observation_data*)\n\x14\x43ompressionTypeProto\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03PNG\x10\x01*@\n\x14ObservationTypeProto\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0f\n\x0bGOAL_SIGNAL\x10\x01\"\x04\x08\x02\x10\x02\"\x04\x08\x03\x10\x03\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - -_COMPRESSIONTYPEPROTO = _descriptor.EnumDescriptor( - name='CompressionTypeProto', - full_name='communicator_objects.CompressionTypeProto', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='NONE', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PNG', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=480, - serialized_end=521, -) -_sym_db.RegisterEnumDescriptor(_COMPRESSIONTYPEPROTO) - -CompressionTypeProto = enum_type_wrapper.EnumTypeWrapper(_COMPRESSIONTYPEPROTO) -_OBSERVATIONTYPEPROTO = _descriptor.EnumDescriptor( - name='ObservationTypeProto', - full_name='communicator_objects.ObservationTypeProto', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='DEFAULT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GOAL_SIGNAL', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=523, - serialized_end=587, -) -_sym_db.RegisterEnumDescriptor(_OBSERVATIONTYPEPROTO) - -ObservationTypeProto = enum_type_wrapper.EnumTypeWrapper(_OBSERVATIONTYPEPROTO) -NONE = 0 -PNG = 1 -DEFAULT = 0 -GOAL_SIGNAL = 1 - - - -_OBSERVATIONPROTO_FLOATDATA = _descriptor.Descriptor( - name='FloatData', - full_name='communicator_objects.ObservationProto.FloatData', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='data', full_name='communicator_objects.ObservationProto.FloatData.data', index=0, - number=1, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=433, - serialized_end=458, -) - -_OBSERVATIONPROTO = _descriptor.Descriptor( - name='ObservationProto', - full_name='communicator_objects.ObservationProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='shape', full_name='communicator_objects.ObservationProto.shape', index=0, - number=1, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='compression_type', full_name='communicator_objects.ObservationProto.compression_type', index=1, - number=2, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='compressed_data', full_name='communicator_objects.ObservationProto.compressed_data', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='float_data', full_name='communicator_objects.ObservationProto.float_data', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='compressed_channel_mapping', full_name='communicator_objects.ObservationProto.compressed_channel_mapping', index=4, - number=5, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dimension_properties', full_name='communicator_objects.ObservationProto.dimension_properties', index=5, - number=6, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='observation_type', full_name='communicator_objects.ObservationProto.observation_type', index=6, - number=7, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='name', full_name='communicator_objects.ObservationProto.name', index=7, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_OBSERVATIONPROTO_FLOATDATA, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='observation_data', full_name='communicator_objects.ObservationProto.observation_data', - index=0, containing_type=None, fields=[]), - ], - serialized_start=79, - serialized_end=478, -) - -_OBSERVATIONPROTO_FLOATDATA.containing_type = _OBSERVATIONPROTO -_OBSERVATIONPROTO.fields_by_name['compression_type'].enum_type = _COMPRESSIONTYPEPROTO -_OBSERVATIONPROTO.fields_by_name['float_data'].message_type = _OBSERVATIONPROTO_FLOATDATA -_OBSERVATIONPROTO.fields_by_name['observation_type'].enum_type = _OBSERVATIONTYPEPROTO -_OBSERVATIONPROTO.oneofs_by_name['observation_data'].fields.append( - _OBSERVATIONPROTO.fields_by_name['compressed_data']) -_OBSERVATIONPROTO.fields_by_name['compressed_data'].containing_oneof = _OBSERVATIONPROTO.oneofs_by_name['observation_data'] -_OBSERVATIONPROTO.oneofs_by_name['observation_data'].fields.append( - _OBSERVATIONPROTO.fields_by_name['float_data']) -_OBSERVATIONPROTO.fields_by_name['float_data'].containing_oneof = _OBSERVATIONPROTO.oneofs_by_name['observation_data'] -DESCRIPTOR.message_types_by_name['ObservationProto'] = _OBSERVATIONPROTO -DESCRIPTOR.enum_types_by_name['CompressionTypeProto'] = _COMPRESSIONTYPEPROTO -DESCRIPTOR.enum_types_by_name['ObservationTypeProto'] = _OBSERVATIONTYPEPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -ObservationProto = _reflection.GeneratedProtocolMessageType('ObservationProto', (_message.Message,), dict( - - FloatData = _reflection.GeneratedProtocolMessageType('FloatData', (_message.Message,), dict( - DESCRIPTOR = _OBSERVATIONPROTO_FLOATDATA, - __module__ = 'mlagents_envs.communicator_objects.observation_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.ObservationProto.FloatData) - )) - , - DESCRIPTOR = _OBSERVATIONPROTO, - __module__ = 'mlagents_envs.communicator_objects.observation_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.ObservationProto) - )) -_sym_db.RegisterMessage(ObservationProto) -_sym_db.RegisterMessage(ObservationProto.FloatData) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/observation_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/observation_pb2.pyi deleted file mode 100644 index 6427c50..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/observation_pb2.pyi +++ /dev/null @@ -1,122 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, - EnumDescriptor as google___protobuf___descriptor___EnumDescriptor, -) - -from google.protobuf.internal.containers import ( - RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - Iterable as typing___Iterable, - List as typing___List, - Optional as typing___Optional, - Text as typing___Text, - Tuple as typing___Tuple, - cast as typing___cast, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int -builtin___str = str - - -class CompressionTypeProto(builtin___int): - DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... - @classmethod - def Name(cls, number: builtin___int) -> builtin___str: ... - @classmethod - def Value(cls, name: builtin___str) -> 'CompressionTypeProto': ... - @classmethod - def keys(cls) -> typing___List[builtin___str]: ... - @classmethod - def values(cls) -> typing___List['CompressionTypeProto']: ... - @classmethod - def items(cls) -> typing___List[typing___Tuple[builtin___str, 'CompressionTypeProto']]: ... - NONE = typing___cast('CompressionTypeProto', 0) - PNG = typing___cast('CompressionTypeProto', 1) -NONE = typing___cast('CompressionTypeProto', 0) -PNG = typing___cast('CompressionTypeProto', 1) - -class ObservationTypeProto(builtin___int): - DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... - @classmethod - def Name(cls, number: builtin___int) -> builtin___str: ... - @classmethod - def Value(cls, name: builtin___str) -> 'ObservationTypeProto': ... - @classmethod - def keys(cls) -> typing___List[builtin___str]: ... - @classmethod - def values(cls) -> typing___List['ObservationTypeProto']: ... - @classmethod - def items(cls) -> typing___List[typing___Tuple[builtin___str, 'ObservationTypeProto']]: ... - DEFAULT = typing___cast('ObservationTypeProto', 0) - GOAL_SIGNAL = typing___cast('ObservationTypeProto', 1) -DEFAULT = typing___cast('ObservationTypeProto', 0) -GOAL_SIGNAL = typing___cast('ObservationTypeProto', 1) - -class ObservationProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class FloatData(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - data = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] - - def __init__(self, - *, - data : typing___Optional[typing___Iterable[builtin___float]] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> ObservationProto.FloatData: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"data"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"data",b"data"]) -> None: ... - - shape = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] - compression_type = ... # type: CompressionTypeProto - compressed_data = ... # type: builtin___bytes - compressed_channel_mapping = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] - dimension_properties = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] - observation_type = ... # type: ObservationTypeProto - name = ... # type: typing___Text - - @property - def float_data(self) -> ObservationProto.FloatData: ... - - def __init__(self, - *, - shape : typing___Optional[typing___Iterable[builtin___int]] = None, - compression_type : typing___Optional[CompressionTypeProto] = None, - compressed_data : typing___Optional[builtin___bytes] = None, - float_data : typing___Optional[ObservationProto.FloatData] = None, - compressed_channel_mapping : typing___Optional[typing___Iterable[builtin___int]] = None, - dimension_properties : typing___Optional[typing___Iterable[builtin___int]] = None, - observation_type : typing___Optional[ObservationTypeProto] = None, - name : typing___Optional[typing___Text] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> ObservationProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"compressed_data",u"float_data",u"observation_data"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"compressed_channel_mapping",u"compressed_data",u"compression_type",u"dimension_properties",u"float_data",u"name",u"observation_data",u"observation_type",u"shape"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"compressed_data",b"compressed_data",u"float_data",b"float_data",u"observation_data",b"observation_data"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"compressed_channel_mapping",b"compressed_channel_mapping",u"compressed_data",b"compressed_data",u"compression_type",b"compression_type",u"dimension_properties",b"dimension_properties",u"float_data",b"float_data",u"name",b"name",u"observation_data",b"observation_data",u"observation_type",b"observation_type",u"shape",b"shape"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions___Literal[u"observation_data",b"observation_data"]) -> typing_extensions___Literal["compressed_data","float_data"]: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/space_type_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/space_type_pb2.py deleted file mode 100644 index df62c23..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/space_type_pb2.py +++ /dev/null @@ -1,59 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/space_type.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/space_type.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n3mlagents_envs/communicator_objects/space_type.proto\x12\x14\x63ommunicator_objects*.\n\x0eSpaceTypeProto\x12\x0c\n\x08\x64iscrete\x10\x00\x12\x0e\n\ncontinuous\x10\x01\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - -_SPACETYPEPROTO = _descriptor.EnumDescriptor( - name='SpaceTypeProto', - full_name='communicator_objects.SpaceTypeProto', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='discrete', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='continuous', index=1, number=1, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=77, - serialized_end=123, -) -_sym_db.RegisterEnumDescriptor(_SPACETYPEPROTO) - -SpaceTypeProto = enum_type_wrapper.EnumTypeWrapper(_SPACETYPEPROTO) -discrete = 0 -continuous = 1 - - -DESCRIPTOR.enum_types_by_name['SpaceTypeProto'] = _SPACETYPEPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/space_type_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/space_type_pb2.pyi deleted file mode 100644 index 4527ffc..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/space_type_pb2.pyi +++ /dev/null @@ -1,37 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - EnumDescriptor as google___protobuf___descriptor___EnumDescriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - List as typing___List, - Tuple as typing___Tuple, - cast as typing___cast, -) - - -builtin___int = int -builtin___str = str - - -class SpaceTypeProto(builtin___int): - DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... - @classmethod - def Name(cls, number: builtin___int) -> builtin___str: ... - @classmethod - def Value(cls, name: builtin___str) -> 'SpaceTypeProto': ... - @classmethod - def keys(cls) -> typing___List[builtin___str]: ... - @classmethod - def values(cls) -> typing___List['SpaceTypeProto']: ... - @classmethod - def items(cls) -> typing___List[typing___Tuple[builtin___str, 'SpaceTypeProto']]: ... - discrete = typing___cast('SpaceTypeProto', 0) - continuous = typing___cast('SpaceTypeProto', 1) -discrete = typing___cast('SpaceTypeProto', 0) -continuous = typing___cast('SpaceTypeProto', 1) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/training_analytics_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/training_analytics_pb2.py deleted file mode 100644 index 2701dac..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/training_analytics_pb2.py +++ /dev/null @@ -1,257 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/training_analytics.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/training_analytics.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n;mlagents_envs/communicator_objects/training_analytics.proto\x12\x14\x63ommunicator_objects\"\xee\x01\n\x1eTrainingEnvironmentInitialized\x12\x18\n\x10mlagents_version\x18\x01 \x01(\t\x12\x1d\n\x15mlagents_envs_version\x18\x02 \x01(\t\x12\x16\n\x0epython_version\x18\x03 \x01(\t\x12\x15\n\rtorch_version\x18\x04 \x01(\t\x12\x19\n\x11torch_device_type\x18\x05 \x01(\t\x12\x10\n\x08num_envs\x18\x06 \x01(\x05\x12\"\n\x1anum_environment_parameters\x18\x07 \x01(\x05\x12\x13\n\x0brun_options\x18\x08 \x01(\t\"\xbd\x03\n\x1bTrainingBehaviorInitialized\x12\x15\n\rbehavior_name\x18\x01 \x01(\t\x12\x14\n\x0ctrainer_type\x18\x02 \x01(\t\x12 \n\x18\x65xtrinsic_reward_enabled\x18\x03 \x01(\x08\x12\x1b\n\x13gail_reward_enabled\x18\x04 \x01(\x08\x12 \n\x18\x63uriosity_reward_enabled\x18\x05 \x01(\x08\x12\x1a\n\x12rnd_reward_enabled\x18\x06 \x01(\x08\x12\"\n\x1a\x62\x65havioral_cloning_enabled\x18\x07 \x01(\x08\x12\x19\n\x11recurrent_enabled\x18\x08 \x01(\x08\x12\x16\n\x0evisual_encoder\x18\t \x01(\t\x12\x1a\n\x12num_network_layers\x18\n \x01(\x05\x12 \n\x18num_network_hidden_units\x18\x0b \x01(\x05\x12\x18\n\x10trainer_threaded\x18\x0c \x01(\x08\x12\x19\n\x11self_play_enabled\x18\r \x01(\x08\x12\x1a\n\x12\x63urriculum_enabled\x18\x0e \x01(\x08\x12\x0e\n\x06\x63onfig\x18\x0f \x01(\tB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') -) - - - - -_TRAININGENVIRONMENTINITIALIZED = _descriptor.Descriptor( - name='TrainingEnvironmentInitialized', - full_name='communicator_objects.TrainingEnvironmentInitialized', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='mlagents_version', full_name='communicator_objects.TrainingEnvironmentInitialized.mlagents_version', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mlagents_envs_version', full_name='communicator_objects.TrainingEnvironmentInitialized.mlagents_envs_version', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='python_version', full_name='communicator_objects.TrainingEnvironmentInitialized.python_version', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='torch_version', full_name='communicator_objects.TrainingEnvironmentInitialized.torch_version', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='torch_device_type', full_name='communicator_objects.TrainingEnvironmentInitialized.torch_device_type', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_envs', full_name='communicator_objects.TrainingEnvironmentInitialized.num_envs', index=5, - number=6, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_environment_parameters', full_name='communicator_objects.TrainingEnvironmentInitialized.num_environment_parameters', index=6, - number=7, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='run_options', full_name='communicator_objects.TrainingEnvironmentInitialized.run_options', index=7, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=86, - serialized_end=324, -) - - -_TRAININGBEHAVIORINITIALIZED = _descriptor.Descriptor( - name='TrainingBehaviorInitialized', - full_name='communicator_objects.TrainingBehaviorInitialized', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='behavior_name', full_name='communicator_objects.TrainingBehaviorInitialized.behavior_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='trainer_type', full_name='communicator_objects.TrainingBehaviorInitialized.trainer_type', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='extrinsic_reward_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.extrinsic_reward_enabled', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='gail_reward_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.gail_reward_enabled', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='curiosity_reward_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.curiosity_reward_enabled', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='rnd_reward_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.rnd_reward_enabled', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='behavioral_cloning_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.behavioral_cloning_enabled', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='recurrent_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.recurrent_enabled', index=7, - number=8, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='visual_encoder', full_name='communicator_objects.TrainingBehaviorInitialized.visual_encoder', index=8, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_network_layers', full_name='communicator_objects.TrainingBehaviorInitialized.num_network_layers', index=9, - number=10, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_network_hidden_units', full_name='communicator_objects.TrainingBehaviorInitialized.num_network_hidden_units', index=10, - number=11, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='trainer_threaded', full_name='communicator_objects.TrainingBehaviorInitialized.trainer_threaded', index=11, - number=12, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='self_play_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.self_play_enabled', index=12, - number=13, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='curriculum_enabled', full_name='communicator_objects.TrainingBehaviorInitialized.curriculum_enabled', index=13, - number=14, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='config', full_name='communicator_objects.TrainingBehaviorInitialized.config', index=14, - number=15, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=327, - serialized_end=772, -) - -DESCRIPTOR.message_types_by_name['TrainingEnvironmentInitialized'] = _TRAININGENVIRONMENTINITIALIZED -DESCRIPTOR.message_types_by_name['TrainingBehaviorInitialized'] = _TRAININGBEHAVIORINITIALIZED -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -TrainingEnvironmentInitialized = _reflection.GeneratedProtocolMessageType('TrainingEnvironmentInitialized', (_message.Message,), dict( - DESCRIPTOR = _TRAININGENVIRONMENTINITIALIZED, - __module__ = 'mlagents_envs.communicator_objects.training_analytics_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.TrainingEnvironmentInitialized) - )) -_sym_db.RegisterMessage(TrainingEnvironmentInitialized) - -TrainingBehaviorInitialized = _reflection.GeneratedProtocolMessageType('TrainingBehaviorInitialized', (_message.Message,), dict( - DESCRIPTOR = _TRAININGBEHAVIORINITIALIZED, - __module__ = 'mlagents_envs.communicator_objects.training_analytics_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.TrainingBehaviorInitialized) - )) -_sym_db.RegisterMessage(TrainingBehaviorInitialized) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/training_analytics_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/training_analytics_pb2.pyi deleted file mode 100644 index 53709ca..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/training_analytics_pb2.pyi +++ /dev/null @@ -1,101 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from typing import ( - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class TrainingEnvironmentInitialized(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - mlagents_version = ... # type: typing___Text - mlagents_envs_version = ... # type: typing___Text - python_version = ... # type: typing___Text - torch_version = ... # type: typing___Text - torch_device_type = ... # type: typing___Text - num_envs = ... # type: builtin___int - num_environment_parameters = ... # type: builtin___int - run_options = ... # type: typing___Text - - def __init__(self, - *, - mlagents_version : typing___Optional[typing___Text] = None, - mlagents_envs_version : typing___Optional[typing___Text] = None, - python_version : typing___Optional[typing___Text] = None, - torch_version : typing___Optional[typing___Text] = None, - torch_device_type : typing___Optional[typing___Text] = None, - num_envs : typing___Optional[builtin___int] = None, - num_environment_parameters : typing___Optional[builtin___int] = None, - run_options : typing___Optional[typing___Text] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> TrainingEnvironmentInitialized: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"mlagents_envs_version",u"mlagents_version",u"num_environment_parameters",u"num_envs",u"python_version",u"run_options",u"torch_device_type",u"torch_version"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"mlagents_envs_version",b"mlagents_envs_version",u"mlagents_version",b"mlagents_version",u"num_environment_parameters",b"num_environment_parameters",u"num_envs",b"num_envs",u"python_version",b"python_version",u"run_options",b"run_options",u"torch_device_type",b"torch_device_type",u"torch_version",b"torch_version"]) -> None: ... - -class TrainingBehaviorInitialized(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - behavior_name = ... # type: typing___Text - trainer_type = ... # type: typing___Text - extrinsic_reward_enabled = ... # type: builtin___bool - gail_reward_enabled = ... # type: builtin___bool - curiosity_reward_enabled = ... # type: builtin___bool - rnd_reward_enabled = ... # type: builtin___bool - behavioral_cloning_enabled = ... # type: builtin___bool - recurrent_enabled = ... # type: builtin___bool - visual_encoder = ... # type: typing___Text - num_network_layers = ... # type: builtin___int - num_network_hidden_units = ... # type: builtin___int - trainer_threaded = ... # type: builtin___bool - self_play_enabled = ... # type: builtin___bool - curriculum_enabled = ... # type: builtin___bool - config = ... # type: typing___Text - - def __init__(self, - *, - behavior_name : typing___Optional[typing___Text] = None, - trainer_type : typing___Optional[typing___Text] = None, - extrinsic_reward_enabled : typing___Optional[builtin___bool] = None, - gail_reward_enabled : typing___Optional[builtin___bool] = None, - curiosity_reward_enabled : typing___Optional[builtin___bool] = None, - rnd_reward_enabled : typing___Optional[builtin___bool] = None, - behavioral_cloning_enabled : typing___Optional[builtin___bool] = None, - recurrent_enabled : typing___Optional[builtin___bool] = None, - visual_encoder : typing___Optional[typing___Text] = None, - num_network_layers : typing___Optional[builtin___int] = None, - num_network_hidden_units : typing___Optional[builtin___int] = None, - trainer_threaded : typing___Optional[builtin___bool] = None, - self_play_enabled : typing___Optional[builtin___bool] = None, - curriculum_enabled : typing___Optional[builtin___bool] = None, - config : typing___Optional[typing___Text] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> TrainingBehaviorInitialized: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"behavior_name",u"behavioral_cloning_enabled",u"config",u"curiosity_reward_enabled",u"curriculum_enabled",u"extrinsic_reward_enabled",u"gail_reward_enabled",u"num_network_hidden_units",u"num_network_layers",u"recurrent_enabled",u"rnd_reward_enabled",u"self_play_enabled",u"trainer_threaded",u"trainer_type",u"visual_encoder"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"behavior_name",b"behavior_name",u"behavioral_cloning_enabled",b"behavioral_cloning_enabled",u"config",b"config",u"curiosity_reward_enabled",b"curiosity_reward_enabled",u"curriculum_enabled",b"curriculum_enabled",u"extrinsic_reward_enabled",b"extrinsic_reward_enabled",u"gail_reward_enabled",b"gail_reward_enabled",u"num_network_hidden_units",b"num_network_hidden_units",u"num_network_layers",b"num_network_layers",u"recurrent_enabled",b"recurrent_enabled",u"rnd_reward_enabled",b"rnd_reward_enabled",u"self_play_enabled",b"self_play_enabled",u"trainer_threaded",b"trainer_threaded",u"trainer_type",b"trainer_type",u"visual_encoder",b"visual_encoder"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_input_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_input_pb2.py deleted file mode 100644 index 6c4a24f..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_input_pb2.py +++ /dev/null @@ -1,83 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_input.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import unity_rl_input_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__rl__input__pb2 -from mlagents_envs.communicator_objects import unity_rl_initialization_input_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__rl__initialization__input__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_input.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n4mlagents_envs/communicator_objects/unity_input.proto\x12\x14\x63ommunicator_objects\x1a\x37mlagents_envs/communicator_objects/unity_rl_input.proto\x1a\x46mlagents_envs/communicator_objects/unity_rl_initialization_input.proto\"\xa4\x01\n\x0fUnityInputProto\x12\x39\n\x08rl_input\x18\x01 \x01(\x0b\x32\'.communicator_objects.UnityRLInputProto\x12V\n\x17rl_initialization_input\x18\x02 \x01(\x0b\x32\x35.communicator_objects.UnityRLInitializationInputProtoB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_unity__rl__input__pb2.DESCRIPTOR,mlagents__envs_dot_communicator__objects_dot_unity__rl__initialization__input__pb2.DESCRIPTOR,]) - - - - -_UNITYINPUTPROTO = _descriptor.Descriptor( - name='UnityInputProto', - full_name='communicator_objects.UnityInputProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='rl_input', full_name='communicator_objects.UnityInputProto.rl_input', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='rl_initialization_input', full_name='communicator_objects.UnityInputProto.rl_initialization_input', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=208, - serialized_end=372, -) - -_UNITYINPUTPROTO.fields_by_name['rl_input'].message_type = mlagents__envs_dot_communicator__objects_dot_unity__rl__input__pb2._UNITYRLINPUTPROTO -_UNITYINPUTPROTO.fields_by_name['rl_initialization_input'].message_type = mlagents__envs_dot_communicator__objects_dot_unity__rl__initialization__input__pb2._UNITYRLINITIALIZATIONINPUTPROTO -DESCRIPTOR.message_types_by_name['UnityInputProto'] = _UNITYINPUTPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityInputProto = _reflection.GeneratedProtocolMessageType('UnityInputProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYINPUTPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_input_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityInputProto) - )) -_sym_db.RegisterMessage(UnityInputProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_input_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_input_pb2.pyi deleted file mode 100644 index 937acf1..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_input_pb2.pyi +++ /dev/null @@ -1,57 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.unity_rl_initialization_input_pb2 import ( - UnityRLInitializationInputProto as mlagents_envs___communicator_objects___unity_rl_initialization_input_pb2___UnityRLInitializationInputProto, -) - -from mlagents_envs.communicator_objects.unity_rl_input_pb2 import ( - UnityRLInputProto as mlagents_envs___communicator_objects___unity_rl_input_pb2___UnityRLInputProto, -) - -from typing import ( - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityInputProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - - @property - def rl_input(self) -> mlagents_envs___communicator_objects___unity_rl_input_pb2___UnityRLInputProto: ... - - @property - def rl_initialization_input(self) -> mlagents_envs___communicator_objects___unity_rl_initialization_input_pb2___UnityRLInitializationInputProto: ... - - def __init__(self, - *, - rl_input : typing___Optional[mlagents_envs___communicator_objects___unity_rl_input_pb2___UnityRLInputProto] = None, - rl_initialization_input : typing___Optional[mlagents_envs___communicator_objects___unity_rl_initialization_input_pb2___UnityRLInitializationInputProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityInputProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",u"rl_input"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",u"rl_input"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",b"rl_initialization_input",u"rl_input",b"rl_input"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_input",b"rl_initialization_input",u"rl_input",b"rl_input"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_message_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_message_pb2.py deleted file mode 100644 index e01b522..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_message_pb2.py +++ /dev/null @@ -1,92 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_message.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import unity_output_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__output__pb2 -from mlagents_envs.communicator_objects import unity_input_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__input__pb2 -from mlagents_envs.communicator_objects import header_pb2 as mlagents__envs_dot_communicator__objects_dot_header__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_message.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n6mlagents_envs/communicator_objects/unity_message.proto\x12\x14\x63ommunicator_objects\x1a\x35mlagents_envs/communicator_objects/unity_output.proto\x1a\x34mlagents_envs/communicator_objects/unity_input.proto\x1a/mlagents_envs/communicator_objects/header.proto\"\xc0\x01\n\x11UnityMessageProto\x12\x31\n\x06header\x18\x01 \x01(\x0b\x32!.communicator_objects.HeaderProto\x12<\n\x0cunity_output\x18\x02 \x01(\x0b\x32&.communicator_objects.UnityOutputProto\x12:\n\x0bunity_input\x18\x03 \x01(\x0b\x32%.communicator_objects.UnityInputProtoB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_unity__output__pb2.DESCRIPTOR,mlagents__envs_dot_communicator__objects_dot_unity__input__pb2.DESCRIPTOR,mlagents__envs_dot_communicator__objects_dot_header__pb2.DESCRIPTOR,]) - - - - -_UNITYMESSAGEPROTO = _descriptor.Descriptor( - name='UnityMessageProto', - full_name='communicator_objects.UnityMessageProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='header', full_name='communicator_objects.UnityMessageProto.header', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unity_output', full_name='communicator_objects.UnityMessageProto.unity_output', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unity_input', full_name='communicator_objects.UnityMessageProto.unity_input', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=239, - serialized_end=431, -) - -_UNITYMESSAGEPROTO.fields_by_name['header'].message_type = mlagents__envs_dot_communicator__objects_dot_header__pb2._HEADERPROTO -_UNITYMESSAGEPROTO.fields_by_name['unity_output'].message_type = mlagents__envs_dot_communicator__objects_dot_unity__output__pb2._UNITYOUTPUTPROTO -_UNITYMESSAGEPROTO.fields_by_name['unity_input'].message_type = mlagents__envs_dot_communicator__objects_dot_unity__input__pb2._UNITYINPUTPROTO -DESCRIPTOR.message_types_by_name['UnityMessageProto'] = _UNITYMESSAGEPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityMessageProto = _reflection.GeneratedProtocolMessageType('UnityMessageProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYMESSAGEPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_message_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityMessageProto) - )) -_sym_db.RegisterMessage(UnityMessageProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_message_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_message_pb2.pyi deleted file mode 100644 index d8f85db..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_message_pb2.pyi +++ /dev/null @@ -1,65 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.header_pb2 import ( - HeaderProto as mlagents_envs___communicator_objects___header_pb2___HeaderProto, -) - -from mlagents_envs.communicator_objects.unity_input_pb2 import ( - UnityInputProto as mlagents_envs___communicator_objects___unity_input_pb2___UnityInputProto, -) - -from mlagents_envs.communicator_objects.unity_output_pb2 import ( - UnityOutputProto as mlagents_envs___communicator_objects___unity_output_pb2___UnityOutputProto, -) - -from typing import ( - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityMessageProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - - @property - def header(self) -> mlagents_envs___communicator_objects___header_pb2___HeaderProto: ... - - @property - def unity_output(self) -> mlagents_envs___communicator_objects___unity_output_pb2___UnityOutputProto: ... - - @property - def unity_input(self) -> mlagents_envs___communicator_objects___unity_input_pb2___UnityInputProto: ... - - def __init__(self, - *, - header : typing___Optional[mlagents_envs___communicator_objects___header_pb2___HeaderProto] = None, - unity_output : typing___Optional[mlagents_envs___communicator_objects___unity_output_pb2___UnityOutputProto] = None, - unity_input : typing___Optional[mlagents_envs___communicator_objects___unity_input_pb2___UnityInputProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityMessageProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"header",u"unity_input",u"unity_output"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"header",u"unity_input",u"unity_output"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"header",b"header",u"unity_input",b"unity_input",u"unity_output",b"unity_output"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"header",b"header",u"unity_input",b"unity_input",u"unity_output",b"unity_output"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_output_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_output_pb2.py deleted file mode 100644 index 400fdb0..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_output_pb2.py +++ /dev/null @@ -1,83 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_output.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import unity_rl_output_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__rl__output__pb2 -from mlagents_envs.communicator_objects import unity_rl_initialization_output_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__rl__initialization__output__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_output.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n5mlagents_envs/communicator_objects/unity_output.proto\x12\x14\x63ommunicator_objects\x1a\x38mlagents_envs/communicator_objects/unity_rl_output.proto\x1aGmlagents_envs/communicator_objects/unity_rl_initialization_output.proto\"\xa9\x01\n\x10UnityOutputProto\x12;\n\trl_output\x18\x01 \x01(\x0b\x32(.communicator_objects.UnityRLOutputProto\x12X\n\x18rl_initialization_output\x18\x02 \x01(\x0b\x32\x36.communicator_objects.UnityRLInitializationOutputProtoB%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_unity__rl__output__pb2.DESCRIPTOR,mlagents__envs_dot_communicator__objects_dot_unity__rl__initialization__output__pb2.DESCRIPTOR,]) - - - - -_UNITYOUTPUTPROTO = _descriptor.Descriptor( - name='UnityOutputProto', - full_name='communicator_objects.UnityOutputProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='rl_output', full_name='communicator_objects.UnityOutputProto.rl_output', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='rl_initialization_output', full_name='communicator_objects.UnityOutputProto.rl_initialization_output', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=211, - serialized_end=380, -) - -_UNITYOUTPUTPROTO.fields_by_name['rl_output'].message_type = mlagents__envs_dot_communicator__objects_dot_unity__rl__output__pb2._UNITYRLOUTPUTPROTO -_UNITYOUTPUTPROTO.fields_by_name['rl_initialization_output'].message_type = mlagents__envs_dot_communicator__objects_dot_unity__rl__initialization__output__pb2._UNITYRLINITIALIZATIONOUTPUTPROTO -DESCRIPTOR.message_types_by_name['UnityOutputProto'] = _UNITYOUTPUTPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityOutputProto = _reflection.GeneratedProtocolMessageType('UnityOutputProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYOUTPUTPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_output_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityOutputProto) - )) -_sym_db.RegisterMessage(UnityOutputProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_output_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_output_pb2.pyi deleted file mode 100644 index 55950f7..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_output_pb2.pyi +++ /dev/null @@ -1,57 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.unity_rl_initialization_output_pb2 import ( - UnityRLInitializationOutputProto as mlagents_envs___communicator_objects___unity_rl_initialization_output_pb2___UnityRLInitializationOutputProto, -) - -from mlagents_envs.communicator_objects.unity_rl_output_pb2 import ( - UnityRLOutputProto as mlagents_envs___communicator_objects___unity_rl_output_pb2___UnityRLOutputProto, -) - -from typing import ( - Optional as typing___Optional, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityOutputProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - - @property - def rl_output(self) -> mlagents_envs___communicator_objects___unity_rl_output_pb2___UnityRLOutputProto: ... - - @property - def rl_initialization_output(self) -> mlagents_envs___communicator_objects___unity_rl_initialization_output_pb2___UnityRLInitializationOutputProto: ... - - def __init__(self, - *, - rl_output : typing___Optional[mlagents_envs___communicator_objects___unity_rl_output_pb2___UnityRLOutputProto] = None, - rl_initialization_output : typing___Optional[mlagents_envs___communicator_objects___unity_rl_initialization_output_pb2___UnityRLInitializationOutputProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityOutputProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",u"rl_output"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",u"rl_output"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",b"rl_initialization_output",u"rl_output",b"rl_output"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"rl_initialization_output",b"rl_initialization_output",u"rl_output",b"rl_output"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_input_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_input_pb2.py deleted file mode 100644 index d111397..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_input_pb2.py +++ /dev/null @@ -1,102 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_rl_initialization_input.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import capabilities_pb2 as mlagents__envs_dot_communicator__objects_dot_capabilities__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_rl_initialization_input.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\nFmlagents_envs/communicator_objects/unity_rl_initialization_input.proto\x12\x14\x63ommunicator_objects\x1a\x35mlagents_envs/communicator_objects/capabilities.proto\"\xc0\x01\n\x1fUnityRLInitializationInputProto\x12\x0c\n\x04seed\x18\x01 \x01(\x05\x12\x1d\n\x15\x63ommunication_version\x18\x02 \x01(\t\x12\x17\n\x0fpackage_version\x18\x03 \x01(\t\x12\x44\n\x0c\x63\x61pabilities\x18\x04 \x01(\x0b\x32..communicator_objects.UnityRLCapabilitiesProto\x12\x11\n\tnum_areas\x18\x05 \x01(\x05\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_capabilities__pb2.DESCRIPTOR,]) - - - - -_UNITYRLINITIALIZATIONINPUTPROTO = _descriptor.Descriptor( - name='UnityRLInitializationInputProto', - full_name='communicator_objects.UnityRLInitializationInputProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='seed', full_name='communicator_objects.UnityRLInitializationInputProto.seed', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='communication_version', full_name='communicator_objects.UnityRLInitializationInputProto.communication_version', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='package_version', full_name='communicator_objects.UnityRLInitializationInputProto.package_version', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='capabilities', full_name='communicator_objects.UnityRLInitializationInputProto.capabilities', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_areas', full_name='communicator_objects.UnityRLInitializationInputProto.num_areas', index=4, - number=5, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=152, - serialized_end=344, -) - -_UNITYRLINITIALIZATIONINPUTPROTO.fields_by_name['capabilities'].message_type = mlagents__envs_dot_communicator__objects_dot_capabilities__pb2._UNITYRLCAPABILITIESPROTO -DESCRIPTOR.message_types_by_name['UnityRLInitializationInputProto'] = _UNITYRLINITIALIZATIONINPUTPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityRLInitializationInputProto = _reflection.GeneratedProtocolMessageType('UnityRLInitializationInputProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYRLINITIALIZATIONINPUTPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_initialization_input_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLInitializationInputProto) - )) -_sym_db.RegisterMessage(UnityRLInitializationInputProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_input_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_input_pb2.pyi deleted file mode 100644 index f502997..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_input_pb2.pyi +++ /dev/null @@ -1,58 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.capabilities_pb2 import ( - UnityRLCapabilitiesProto as mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto, -) - -from typing import ( - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityRLInitializationInputProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - seed = ... # type: builtin___int - communication_version = ... # type: typing___Text - package_version = ... # type: typing___Text - num_areas = ... # type: builtin___int - - @property - def capabilities(self) -> mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto: ... - - def __init__(self, - *, - seed : typing___Optional[builtin___int] = None, - communication_version : typing___Optional[typing___Text] = None, - package_version : typing___Optional[typing___Text] = None, - capabilities : typing___Optional[mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto] = None, - num_areas : typing___Optional[builtin___int] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLInitializationInputProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"capabilities"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"capabilities",u"communication_version",u"num_areas",u"package_version",u"seed"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities",u"communication_version",b"communication_version",u"num_areas",b"num_areas",u"package_version",b"package_version",u"seed",b"seed"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_output_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_output_pb2.py deleted file mode 100644 index 155af96..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_output_pb2.py +++ /dev/null @@ -1,111 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_rl_initialization_output.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import capabilities_pb2 as mlagents__envs_dot_communicator__objects_dot_capabilities__pb2 -from mlagents_envs.communicator_objects import brain_parameters_pb2 as mlagents__envs_dot_communicator__objects_dot_brain__parameters__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_rl_initialization_output.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\nGmlagents_envs/communicator_objects/unity_rl_initialization_output.proto\x12\x14\x63ommunicator_objects\x1a\x35mlagents_envs/communicator_objects/capabilities.proto\x1a\x39mlagents_envs/communicator_objects/brain_parameters.proto\"\x8c\x02\n UnityRLInitializationOutputProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1d\n\x15\x63ommunication_version\x18\x02 \x01(\t\x12\x10\n\x08log_path\x18\x03 \x01(\t\x12\x44\n\x10\x62rain_parameters\x18\x05 \x03(\x0b\x32*.communicator_objects.BrainParametersProto\x12\x17\n\x0fpackage_version\x18\x07 \x01(\t\x12\x44\n\x0c\x63\x61pabilities\x18\x08 \x01(\x0b\x32..communicator_objects.UnityRLCapabilitiesProtoJ\x04\x08\x06\x10\x07\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_capabilities__pb2.DESCRIPTOR,mlagents__envs_dot_communicator__objects_dot_brain__parameters__pb2.DESCRIPTOR,]) - - - - -_UNITYRLINITIALIZATIONOUTPUTPROTO = _descriptor.Descriptor( - name='UnityRLInitializationOutputProto', - full_name='communicator_objects.UnityRLInitializationOutputProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='communicator_objects.UnityRLInitializationOutputProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='communication_version', full_name='communicator_objects.UnityRLInitializationOutputProto.communication_version', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='log_path', full_name='communicator_objects.UnityRLInitializationOutputProto.log_path', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='brain_parameters', full_name='communicator_objects.UnityRLInitializationOutputProto.brain_parameters', index=3, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='package_version', full_name='communicator_objects.UnityRLInitializationOutputProto.package_version', index=4, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='capabilities', full_name='communicator_objects.UnityRLInitializationOutputProto.capabilities', index=5, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=212, - serialized_end=480, -) - -_UNITYRLINITIALIZATIONOUTPUTPROTO.fields_by_name['brain_parameters'].message_type = mlagents__envs_dot_communicator__objects_dot_brain__parameters__pb2._BRAINPARAMETERSPROTO -_UNITYRLINITIALIZATIONOUTPUTPROTO.fields_by_name['capabilities'].message_type = mlagents__envs_dot_communicator__objects_dot_capabilities__pb2._UNITYRLCAPABILITIESPROTO -DESCRIPTOR.message_types_by_name['UnityRLInitializationOutputProto'] = _UNITYRLINITIALIZATIONOUTPUTPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityRLInitializationOutputProto = _reflection.GeneratedProtocolMessageType('UnityRLInitializationOutputProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYRLINITIALIZATIONOUTPUTPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_initialization_output_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLInitializationOutputProto) - )) -_sym_db.RegisterMessage(UnityRLInitializationOutputProto) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_output_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_output_pb2.pyi deleted file mode 100644 index cce72df..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_initialization_output_pb2.pyi +++ /dev/null @@ -1,71 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.internal.containers import ( - RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.brain_parameters_pb2 import ( - BrainParametersProto as mlagents_envs___communicator_objects___brain_parameters_pb2___BrainParametersProto, -) - -from mlagents_envs.communicator_objects.capabilities_pb2 import ( - UnityRLCapabilitiesProto as mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto, -) - -from typing import ( - Iterable as typing___Iterable, - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityRLInitializationOutputProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - name = ... # type: typing___Text - communication_version = ... # type: typing___Text - log_path = ... # type: typing___Text - package_version = ... # type: typing___Text - - @property - def brain_parameters(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents_envs___communicator_objects___brain_parameters_pb2___BrainParametersProto]: ... - - @property - def capabilities(self) -> mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto: ... - - def __init__(self, - *, - name : typing___Optional[typing___Text] = None, - communication_version : typing___Optional[typing___Text] = None, - log_path : typing___Optional[typing___Text] = None, - brain_parameters : typing___Optional[typing___Iterable[mlagents_envs___communicator_objects___brain_parameters_pb2___BrainParametersProto]] = None, - package_version : typing___Optional[typing___Text] = None, - capabilities : typing___Optional[mlagents_envs___communicator_objects___capabilities_pb2___UnityRLCapabilitiesProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLInitializationOutputProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"capabilities"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"brain_parameters",u"capabilities",u"communication_version",u"log_path",u"name",u"package_version"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"capabilities",b"capabilities"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"brain_parameters",b"brain_parameters",u"capabilities",b"capabilities",u"communication_version",b"communication_version",u"log_path",b"log_path",u"name",b"name",u"package_version",b"package_version"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_input_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_input_pb2.py deleted file mode 100644 index 69e14f3..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_input_pb2.py +++ /dev/null @@ -1,179 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_rl_input.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import agent_action_pb2 as mlagents__envs_dot_communicator__objects_dot_agent__action__pb2 -from mlagents_envs.communicator_objects import command_pb2 as mlagents__envs_dot_communicator__objects_dot_command__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_rl_input.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n7mlagents_envs/communicator_objects/unity_rl_input.proto\x12\x14\x63ommunicator_objects\x1a\x35mlagents_envs/communicator_objects/agent_action.proto\x1a\x30mlagents_envs/communicator_objects/command.proto\"\xfe\x02\n\x11UnityRLInputProto\x12P\n\ragent_actions\x18\x01 \x03(\x0b\x32\x39.communicator_objects.UnityRLInputProto.AgentActionsEntry\x12\x33\n\x07\x63ommand\x18\x04 \x01(\x0e\x32\".communicator_objects.CommandProto\x12\x14\n\x0cside_channel\x18\x05 \x01(\x0c\x1aM\n\x14ListAgentActionProto\x12\x35\n\x05value\x18\x01 \x03(\x0b\x32&.communicator_objects.AgentActionProto\x1aq\n\x11\x41gentActionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12K\n\x05value\x18\x02 \x01(\x0b\x32<.communicator_objects.UnityRLInputProto.ListAgentActionProto:\x02\x38\x01J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_agent__action__pb2.DESCRIPTOR,mlagents__envs_dot_communicator__objects_dot_command__pb2.DESCRIPTOR,]) - - - - -_UNITYRLINPUTPROTO_LISTAGENTACTIONPROTO = _descriptor.Descriptor( - name='ListAgentActionProto', - full_name='communicator_objects.UnityRLInputProto.ListAgentActionProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='communicator_objects.UnityRLInputProto.ListAgentActionProto.value', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=365, - serialized_end=442, -) - -_UNITYRLINPUTPROTO_AGENTACTIONSENTRY = _descriptor.Descriptor( - name='AgentActionsEntry', - full_name='communicator_objects.UnityRLInputProto.AgentActionsEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='communicator_objects.UnityRLInputProto.AgentActionsEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='communicator_objects.UnityRLInputProto.AgentActionsEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=444, - serialized_end=557, -) - -_UNITYRLINPUTPROTO = _descriptor.Descriptor( - name='UnityRLInputProto', - full_name='communicator_objects.UnityRLInputProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='agent_actions', full_name='communicator_objects.UnityRLInputProto.agent_actions', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='command', full_name='communicator_objects.UnityRLInputProto.command', index=1, - number=4, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='side_channel', full_name='communicator_objects.UnityRLInputProto.side_channel', index=2, - number=5, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_UNITYRLINPUTPROTO_LISTAGENTACTIONPROTO, _UNITYRLINPUTPROTO_AGENTACTIONSENTRY, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=187, - serialized_end=569, -) - -_UNITYRLINPUTPROTO_LISTAGENTACTIONPROTO.fields_by_name['value'].message_type = mlagents__envs_dot_communicator__objects_dot_agent__action__pb2._AGENTACTIONPROTO -_UNITYRLINPUTPROTO_LISTAGENTACTIONPROTO.containing_type = _UNITYRLINPUTPROTO -_UNITYRLINPUTPROTO_AGENTACTIONSENTRY.fields_by_name['value'].message_type = _UNITYRLINPUTPROTO_LISTAGENTACTIONPROTO -_UNITYRLINPUTPROTO_AGENTACTIONSENTRY.containing_type = _UNITYRLINPUTPROTO -_UNITYRLINPUTPROTO.fields_by_name['agent_actions'].message_type = _UNITYRLINPUTPROTO_AGENTACTIONSENTRY -_UNITYRLINPUTPROTO.fields_by_name['command'].enum_type = mlagents__envs_dot_communicator__objects_dot_command__pb2._COMMANDPROTO -DESCRIPTOR.message_types_by_name['UnityRLInputProto'] = _UNITYRLINPUTPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityRLInputProto = _reflection.GeneratedProtocolMessageType('UnityRLInputProto', (_message.Message,), dict( - - ListAgentActionProto = _reflection.GeneratedProtocolMessageType('ListAgentActionProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYRLINPUTPROTO_LISTAGENTACTIONPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_input_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLInputProto.ListAgentActionProto) - )) - , - - AgentActionsEntry = _reflection.GeneratedProtocolMessageType('AgentActionsEntry', (_message.Message,), dict( - DESCRIPTOR = _UNITYRLINPUTPROTO_AGENTACTIONSENTRY, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_input_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLInputProto.AgentActionsEntry) - )) - , - DESCRIPTOR = _UNITYRLINPUTPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_input_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLInputProto) - )) -_sym_db.RegisterMessage(UnityRLInputProto) -_sym_db.RegisterMessage(UnityRLInputProto.ListAgentActionProto) -_sym_db.RegisterMessage(UnityRLInputProto.AgentActionsEntry) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -_UNITYRLINPUTPROTO_AGENTACTIONSENTRY.has_options = True -_UNITYRLINPUTPROTO_AGENTACTIONSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_input_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_input_pb2.pyi deleted file mode 100644 index 563f8fa..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_input_pb2.pyi +++ /dev/null @@ -1,105 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.internal.containers import ( - RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.agent_action_pb2 import ( - AgentActionProto as mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto, -) - -from mlagents_envs.communicator_objects.command_pb2 import ( - CommandProto as mlagents_envs___communicator_objects___command_pb2___CommandProto, -) - -from typing import ( - Iterable as typing___Iterable, - Mapping as typing___Mapping, - MutableMapping as typing___MutableMapping, - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityRLInputProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class ListAgentActionProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - - @property - def value(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto]: ... - - def __init__(self, - *, - value : typing___Optional[typing___Iterable[mlagents_envs___communicator_objects___agent_action_pb2___AgentActionProto]] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLInputProto.ListAgentActionProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"value"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> None: ... - - class AgentActionsEntry(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - key = ... # type: typing___Text - - @property - def value(self) -> UnityRLInputProto.ListAgentActionProto: ... - - def __init__(self, - *, - key : typing___Optional[typing___Text] = None, - value : typing___Optional[UnityRLInputProto.ListAgentActionProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLInputProto.AgentActionsEntry: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... - - command = ... # type: mlagents_envs___communicator_objects___command_pb2___CommandProto - side_channel = ... # type: builtin___bytes - - @property - def agent_actions(self) -> typing___MutableMapping[typing___Text, UnityRLInputProto.ListAgentActionProto]: ... - - def __init__(self, - *, - agent_actions : typing___Optional[typing___Mapping[typing___Text, UnityRLInputProto.ListAgentActionProto]] = None, - command : typing___Optional[mlagents_envs___communicator_objects___command_pb2___CommandProto] = None, - side_channel : typing___Optional[builtin___bytes] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLInputProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"agent_actions",u"command",u"side_channel"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"agent_actions",b"agent_actions",u"command",b"command",u"side_channel",b"side_channel"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_output_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_output_pb2.py deleted file mode 100644 index 099d0b0..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_output_pb2.py +++ /dev/null @@ -1,170 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_rl_output.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import agent_info_pb2 as mlagents__envs_dot_communicator__objects_dot_agent__info__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_rl_output.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n8mlagents_envs/communicator_objects/unity_rl_output.proto\x12\x14\x63ommunicator_objects\x1a\x33mlagents_envs/communicator_objects/agent_info.proto\"\xb9\x02\n\x12UnityRLOutputProto\x12L\n\nagentInfos\x18\x02 \x03(\x0b\x32\x38.communicator_objects.UnityRLOutputProto.AgentInfosEntry\x12\x14\n\x0cside_channel\x18\x03 \x01(\x0c\x1aI\n\x12ListAgentInfoProto\x12\x33\n\x05value\x18\x01 \x03(\x0b\x32$.communicator_objects.AgentInfoProto\x1an\n\x0f\x41gentInfosEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12J\n\x05value\x18\x02 \x01(\x0b\x32;.communicator_objects.UnityRLOutputProto.ListAgentInfoProto:\x02\x38\x01J\x04\x08\x01\x10\x02\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_agent__info__pb2.DESCRIPTOR,]) - - - - -_UNITYRLOUTPUTPROTO_LISTAGENTINFOPROTO = _descriptor.Descriptor( - name='ListAgentInfoProto', - full_name='communicator_objects.UnityRLOutputProto.ListAgentInfoProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='communicator_objects.UnityRLOutputProto.ListAgentInfoProto.value', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=258, - serialized_end=331, -) - -_UNITYRLOUTPUTPROTO_AGENTINFOSENTRY = _descriptor.Descriptor( - name='AgentInfosEntry', - full_name='communicator_objects.UnityRLOutputProto.AgentInfosEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='communicator_objects.UnityRLOutputProto.AgentInfosEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='communicator_objects.UnityRLOutputProto.AgentInfosEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=333, - serialized_end=443, -) - -_UNITYRLOUTPUTPROTO = _descriptor.Descriptor( - name='UnityRLOutputProto', - full_name='communicator_objects.UnityRLOutputProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='agentInfos', full_name='communicator_objects.UnityRLOutputProto.agentInfos', index=0, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='side_channel', full_name='communicator_objects.UnityRLOutputProto.side_channel', index=1, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_UNITYRLOUTPUTPROTO_LISTAGENTINFOPROTO, _UNITYRLOUTPUTPROTO_AGENTINFOSENTRY, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=136, - serialized_end=449, -) - -_UNITYRLOUTPUTPROTO_LISTAGENTINFOPROTO.fields_by_name['value'].message_type = mlagents__envs_dot_communicator__objects_dot_agent__info__pb2._AGENTINFOPROTO -_UNITYRLOUTPUTPROTO_LISTAGENTINFOPROTO.containing_type = _UNITYRLOUTPUTPROTO -_UNITYRLOUTPUTPROTO_AGENTINFOSENTRY.fields_by_name['value'].message_type = _UNITYRLOUTPUTPROTO_LISTAGENTINFOPROTO -_UNITYRLOUTPUTPROTO_AGENTINFOSENTRY.containing_type = _UNITYRLOUTPUTPROTO -_UNITYRLOUTPUTPROTO.fields_by_name['agentInfos'].message_type = _UNITYRLOUTPUTPROTO_AGENTINFOSENTRY -DESCRIPTOR.message_types_by_name['UnityRLOutputProto'] = _UNITYRLOUTPUTPROTO -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -UnityRLOutputProto = _reflection.GeneratedProtocolMessageType('UnityRLOutputProto', (_message.Message,), dict( - - ListAgentInfoProto = _reflection.GeneratedProtocolMessageType('ListAgentInfoProto', (_message.Message,), dict( - DESCRIPTOR = _UNITYRLOUTPUTPROTO_LISTAGENTINFOPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_output_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLOutputProto.ListAgentInfoProto) - )) - , - - AgentInfosEntry = _reflection.GeneratedProtocolMessageType('AgentInfosEntry', (_message.Message,), dict( - DESCRIPTOR = _UNITYRLOUTPUTPROTO_AGENTINFOSENTRY, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_output_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLOutputProto.AgentInfosEntry) - )) - , - DESCRIPTOR = _UNITYRLOUTPUTPROTO, - __module__ = 'mlagents_envs.communicator_objects.unity_rl_output_pb2' - # @@protoc_insertion_point(class_scope:communicator_objects.UnityRLOutputProto) - )) -_sym_db.RegisterMessage(UnityRLOutputProto) -_sym_db.RegisterMessage(UnityRLOutputProto.ListAgentInfoProto) -_sym_db.RegisterMessage(UnityRLOutputProto.AgentInfosEntry) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) -_UNITYRLOUTPUTPROTO_AGENTINFOSENTRY.has_options = True -_UNITYRLOUTPUTPROTO_AGENTINFOSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_output_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_output_pb2.pyi deleted file mode 100644 index 0ace7bd..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_rl_output_pb2.pyi +++ /dev/null @@ -1,99 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.descriptor import ( - Descriptor as google___protobuf___descriptor___Descriptor, -) - -from google.protobuf.internal.containers import ( - RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, -) - -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - -from mlagents_envs.communicator_objects.agent_info_pb2 import ( - AgentInfoProto as mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto, -) - -from typing import ( - Iterable as typing___Iterable, - Mapping as typing___Mapping, - MutableMapping as typing___MutableMapping, - Optional as typing___Optional, - Text as typing___Text, -) - -from typing_extensions import ( - Literal as typing_extensions___Literal, -) - - -builtin___bool = bool -builtin___bytes = bytes -builtin___float = float -builtin___int = int - - -class UnityRLOutputProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class ListAgentInfoProto(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - - @property - def value(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto]: ... - - def __init__(self, - *, - value : typing___Optional[typing___Iterable[mlagents_envs___communicator_objects___agent_info_pb2___AgentInfoProto]] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLOutputProto.ListAgentInfoProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"value"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> None: ... - - class AgentInfosEntry(google___protobuf___message___Message): - DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - key = ... # type: typing___Text - - @property - def value(self) -> UnityRLOutputProto.ListAgentInfoProto: ... - - def __init__(self, - *, - key : typing___Optional[typing___Text] = None, - value : typing___Optional[UnityRLOutputProto.ListAgentInfoProto] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLOutputProto.AgentInfosEntry: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... - else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... - - side_channel = ... # type: builtin___bytes - - @property - def agentInfos(self) -> typing___MutableMapping[typing___Text, UnityRLOutputProto.ListAgentInfoProto]: ... - - def __init__(self, - *, - agentInfos : typing___Optional[typing___Mapping[typing___Text, UnityRLOutputProto.ListAgentInfoProto]] = None, - side_channel : typing___Optional[builtin___bytes] = None, - ) -> None: ... - @classmethod - def FromString(cls, s: builtin___bytes) -> UnityRLOutputProto: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"agentInfos",u"side_channel"]) -> None: ... - else: - def ClearField(self, field_name: typing_extensions___Literal[u"agentInfos",b"agentInfos",u"side_channel",b"side_channel"]) -> None: ... diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2.py deleted file mode 100644 index 6053ccc..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2.py +++ /dev/null @@ -1,58 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mlagents_envs/communicator_objects/unity_to_external.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from mlagents_envs.communicator_objects import unity_message_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__message__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mlagents_envs/communicator_objects/unity_to_external.proto', - package='communicator_objects', - syntax='proto3', - serialized_pb=_b('\n:mlagents_envs/communicator_objects/unity_to_external.proto\x12\x14\x63ommunicator_objects\x1a\x36mlagents_envs/communicator_objects/unity_message.proto2v\n\x14UnityToExternalProto\x12^\n\x08\x45xchange\x12\'.communicator_objects.UnityMessageProto\x1a\'.communicator_objects.UnityMessageProto\"\x00\x42%\xaa\x02\"Unity.MLAgents.CommunicatorObjectsb\x06proto3') - , - dependencies=[mlagents__envs_dot_communicator__objects_dot_unity__message__pb2.DESCRIPTOR,]) - - - -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\"Unity.MLAgents.CommunicatorObjects')) - -_UNITYTOEXTERNALPROTO = _descriptor.ServiceDescriptor( - name='UnityToExternalProto', - full_name='communicator_objects.UnityToExternalProto', - file=DESCRIPTOR, - index=0, - options=None, - serialized_start=140, - serialized_end=258, - methods=[ - _descriptor.MethodDescriptor( - name='Exchange', - full_name='communicator_objects.UnityToExternalProto.Exchange', - index=0, - containing_service=None, - input_type=mlagents__envs_dot_communicator__objects_dot_unity__message__pb2._UNITYMESSAGEPROTO, - output_type=mlagents__envs_dot_communicator__objects_dot_unity__message__pb2._UNITYMESSAGEPROTO, - options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_UNITYTOEXTERNALPROTO) - -DESCRIPTOR.services_by_name['UnityToExternalProto'] = _UNITYTOEXTERNALPROTO - -# @@protoc_insertion_point(module_scope) diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2.pyi b/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2.pyi deleted file mode 100644 index 8bba4ff..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2.pyi +++ /dev/null @@ -1,6 +0,0 @@ -# @generated by generate_proto_mypy_stubs.py. Do not edit! -import sys -from google.protobuf.message import ( - Message as google___protobuf___message___Message, -) - diff --git a/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py b/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py deleted file mode 100644 index bedeb10..0000000 --- a/3DOrangeTree/mlagents_envs/communicator_objects/unity_to_external_pb2_grpc.py +++ /dev/null @@ -1,46 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -import grpc - -from mlagents_envs.communicator_objects import unity_message_pb2 as mlagents__envs_dot_communicator__objects_dot_unity__message__pb2 - - -class UnityToExternalProtoStub(object): - # missing associated documentation comment in .proto file - pass - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Exchange = channel.unary_unary( - '/communicator_objects.UnityToExternalProto/Exchange', - request_serializer=mlagents__envs_dot_communicator__objects_dot_unity__message__pb2.UnityMessageProto.SerializeToString, - response_deserializer=mlagents__envs_dot_communicator__objects_dot_unity__message__pb2.UnityMessageProto.FromString, - ) - - -class UnityToExternalProtoServicer(object): - # missing associated documentation comment in .proto file - pass - - def Exchange(self, request, context): - """Sends the academy parameters - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_UnityToExternalProtoServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Exchange': grpc.unary_unary_rpc_method_handler( - servicer.Exchange, - request_deserializer=mlagents__envs_dot_communicator__objects_dot_unity__message__pb2.UnityMessageProto.FromString, - response_serializer=mlagents__envs_dot_communicator__objects_dot_unity__message__pb2.UnityMessageProto.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'communicator_objects.UnityToExternalProto', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) diff --git a/3DOrangeTree/mlagents_envs/env_utils.py b/3DOrangeTree/mlagents_envs/env_utils.py deleted file mode 100644 index 73393d9..0000000 --- a/3DOrangeTree/mlagents_envs/env_utils.py +++ /dev/null @@ -1,130 +0,0 @@ -import glob -import os -import subprocess -from sys import platform -from typing import Optional, List -from mlagents_envs.logging_util import get_logger, DEBUG -from mlagents_envs.exception import UnityEnvironmentException - - -logger = get_logger(__name__) - - -def get_platform(): - """ - returns the platform of the operating system : linux, darwin or win32 - """ - return platform - - -def validate_environment_path(env_path: str) -> Optional[str]: - """ - Strip out executable extensions of the env_path - :param env_path: The path to the executable - """ - env_path = ( - env_path.strip() - .replace(".app", "") - .replace(".exe", "") - .replace(".x86_64", "") - .replace(".x86", "") - ) - true_filename = os.path.basename(os.path.normpath(env_path)) - logger.debug(f"The true file name is {true_filename}") - - if not (glob.glob(env_path) or glob.glob(env_path + ".*")): - return None - - cwd = os.getcwd() - launch_string = None - true_filename = os.path.basename(os.path.normpath(env_path)) - if get_platform() == "linux" or get_platform() == "linux2": - candidates = glob.glob(os.path.join(cwd, env_path) + ".x86_64") - if len(candidates) == 0: - candidates = glob.glob(os.path.join(cwd, env_path) + ".x86") - if len(candidates) == 0: - candidates = glob.glob(env_path + ".x86_64") - if len(candidates) == 0: - candidates = glob.glob(env_path + ".x86") - if len(candidates) == 0: - if os.path.isfile(env_path): - candidates = [env_path] - if len(candidates) > 0: - launch_string = candidates[0] - - elif get_platform() == "darwin": - candidates = glob.glob( - os.path.join(cwd, env_path + ".app", "Contents", "MacOS", true_filename) - ) - if len(candidates) == 0: - candidates = glob.glob( - os.path.join(env_path + ".app", "Contents", "MacOS", true_filename) - ) - if len(candidates) == 0: - candidates = glob.glob( - os.path.join(cwd, env_path + ".app", "Contents", "MacOS", "*") - ) - if len(candidates) == 0: - candidates = glob.glob( - os.path.join(env_path + ".app", "Contents", "MacOS", "*") - ) - if len(candidates) > 0: - launch_string = candidates[0] - elif get_platform() == "win32": - candidates = glob.glob(os.path.join(cwd, env_path + ".exe")) - if len(candidates) == 0: - candidates = glob.glob(env_path + ".exe") - if len(candidates) == 0: - # Look for e.g. 3DBall\UnityEnvironment.exe - crash_handlers = set( - glob.glob(os.path.join(cwd, env_path, "UnityCrashHandler*.exe")) - ) - candidates = [ - c - for c in glob.glob(os.path.join(cwd, env_path, "*.exe")) - if c not in crash_handlers - ] - if len(candidates) > 0: - launch_string = candidates[0] - return launch_string - - -def launch_executable(file_name: str, args: List[str]) -> subprocess.Popen: - """ - Launches a Unity executable and returns the process handle for it. - :param file_name: the name of the executable - :param args: List of string that will be passed as command line arguments - when launching the executable. - """ - launch_string = validate_environment_path(file_name) - if launch_string is None: - raise UnityEnvironmentException( - f"Couldn't launch the {file_name} environment. Provided filename does not match any environments." - ) - else: - logger.debug(f"The launch string is {launch_string}") - logger.debug(f"Running with args {args}") - # Launch Unity environment - subprocess_args = [launch_string] + args - # std_out_option = DEVNULL means the outputs will not be displayed on terminal. - # std_out_option = None is default behavior: the outputs are displayed on terminal. - std_out_option = subprocess.DEVNULL if logger.level > DEBUG else None - try: - return subprocess.Popen( - subprocess_args, - # start_new_session=True means that signals to the parent python process - # (e.g. SIGINT from keyboard interrupt) will not be sent to the new process on POSIX platforms. - # This is generally good since we want the environment to have a chance to shutdown, - # but may be undesirable in come cases; if so, we'll add a command-line toggle. - # Note that on Windows, the CTRL_C signal will still be sent. - start_new_session=True, - stdout=std_out_option, - stderr=std_out_option, - ) - except PermissionError as perm: - # This is likely due to missing read or execute permissions on file. - raise UnityEnvironmentException( - f"Error when trying to launch environment - make sure " - f"permissions are set correctly. For example " - f'"chmod -R 755 {launch_string}"' - ) from perm diff --git a/3DOrangeTree/mlagents_envs/environment.py b/3DOrangeTree/mlagents_envs/environment.py deleted file mode 100644 index 18731a2..0000000 --- a/3DOrangeTree/mlagents_envs/environment.py +++ /dev/null @@ -1,518 +0,0 @@ -import atexit -from distutils.version import StrictVersion - -import numpy as np -import os -import subprocess -from typing import Dict, List, Optional, Tuple, Mapping as MappingType - -import mlagents_envs - -from mlagents_envs.logging_util import get_logger -from mlagents_envs.side_channel.side_channel import SideChannel -from mlagents_envs.side_channel import DefaultTrainingAnalyticsSideChannel -from mlagents_envs.side_channel.side_channel_manager import SideChannelManager -from mlagents_envs import env_utils - -from mlagents_envs.base_env import ( - BaseEnv, - DecisionSteps, - TerminalSteps, - BehaviorSpec, - ActionTuple, - BehaviorName, - AgentId, - BehaviorMapping, -) -from mlagents_envs.timers import timed, hierarchical_timer -from mlagents_envs.exception import ( - UnityEnvironmentException, - UnityActionException, - UnityTimeOutException, - UnityCommunicatorStoppedException, -) - -from mlagents_envs.communicator_objects.command_pb2 import STEP, RESET -from mlagents_envs.rpc_utils import behavior_spec_from_proto, steps_from_proto - -from mlagents_envs.communicator_objects.unity_rl_input_pb2 import UnityRLInputProto -from mlagents_envs.communicator_objects.unity_rl_output_pb2 import UnityRLOutputProto -from mlagents_envs.communicator_objects.agent_action_pb2 import AgentActionProto -from mlagents_envs.communicator_objects.unity_output_pb2 import UnityOutputProto -from mlagents_envs.communicator_objects.capabilities_pb2 import UnityRLCapabilitiesProto -from mlagents_envs.communicator_objects.unity_rl_initialization_input_pb2 import ( - UnityRLInitializationInputProto, -) - -from mlagents_envs.communicator_objects.unity_input_pb2 import UnityInputProto - -from .rpc_communicator import RpcCommunicator -import signal - -logger = get_logger(__name__) - - -class UnityEnvironment(BaseEnv): - # Communication protocol version. - # When connecting to C#, this must be compatible with Academy.k_ApiVersion. - # We follow semantic versioning on the communication version, so existing - # functionality will work as long the major versions match. - # This should be changed whenever a change is made to the communication protocol. - # Revision history: - # * 1.0.0 - initial version - # * 1.1.0 - support concatenated PNGs for compressed observations. - # * 1.2.0 - support compression mapping for stacked compressed observations. - # * 1.3.0 - support action spaces with both continuous and discrete actions. - # * 1.4.0 - support training analytics sent from python trainer to the editor. - # * 1.5.0 - support variable length observation training and multi-agent groups. - API_VERSION = "1.5.0" - - # Default port that the editor listens on. If an environment executable - # isn't specified, this port will be used. - DEFAULT_EDITOR_PORT = 5004 - - # Default base port for environments. Each environment will be offset from this - # by it's worker_id. - BASE_ENVIRONMENT_PORT = 5005 - - # Command line argument used to pass the port to the executable environment. - _PORT_COMMAND_LINE_ARG = "--mlagents-port" - - @staticmethod - def _raise_version_exception(unity_com_ver: str) -> None: - raise UnityEnvironmentException( - f"The communication API version is not compatible between Unity and python. " - f"Python API: {UnityEnvironment.API_VERSION}, Unity API: {unity_com_ver}.\n " - f"Please find the versions that work best together from our release page.\n" - "https://github.com/Unity-Technologies/ml-agents/releases" - ) - - @staticmethod - def _check_communication_compatibility( - unity_com_ver: str, python_api_version: str, unity_package_version: str - ) -> bool: - unity_communicator_version = StrictVersion(unity_com_ver) - api_version = StrictVersion(python_api_version) - if unity_communicator_version.version[0] == 0: - if ( - unity_communicator_version.version[0] != api_version.version[0] - or unity_communicator_version.version[1] != api_version.version[1] - ): - # Minor beta versions differ. - return False - elif unity_communicator_version.version[0] != api_version.version[0]: - # Major versions mismatch. - return False - else: - # Major versions match, so either: - # 1) The versions are identical, in which case there's no compatibility issues - # 2) The Unity version is newer, in which case we'll warn or fail on the Unity side if trying to use - # unsupported features - # 3) The trainer version is newer, in which case new trainer features might be available but unused by C# - # In any of the cases, there's no reason to warn about mismatch here. - logger.info( - f"Connected to Unity environment with package version {unity_package_version} " - f"and communication version {unity_com_ver}" - ) - return True - - @staticmethod - def _get_capabilities_proto() -> UnityRLCapabilitiesProto: - capabilities = UnityRLCapabilitiesProto() - capabilities.baseRLCapabilities = True - capabilities.concatenatedPngObservations = True - capabilities.compressedChannelMapping = True - capabilities.hybridActions = True - capabilities.trainingAnalytics = True - capabilities.variableLengthObservation = True - capabilities.multiAgentGroups = True - return capabilities - - @staticmethod - def _warn_csharp_base_capabilities( - caps: UnityRLCapabilitiesProto, unity_package_ver: str, python_package_ver: str - ) -> None: - if not caps.baseRLCapabilities: - logger.warning( - "WARNING: The Unity process is not running with the expected base Reinforcement Learning" - " capabilities. Please be sure upgrade the Unity Package to a version that is compatible with this " - "python package.\n" - f"Python package version: {python_package_ver}, C# package version: {unity_package_ver}" - f"Please find the versions that work best together from our release page.\n" - "https://github.com/Unity-Technologies/ml-agents/releases" - ) - - def __init__( - self, - file_name: Optional[str] = None, - worker_id: int = 0, - base_port: Optional[int] = None, - seed: int = 0, - no_graphics: bool = False, - timeout_wait: int = 60, - additional_args: Optional[List[str]] = None, - side_channels: Optional[List[SideChannel]] = None, - log_folder: Optional[str] = None, - num_areas: int = 1, - ): - """ - Starts a new unity environment and establishes a connection with the environment. - Notice: Currently communication between Unity and Python takes place over an open socket without authentication. - Ensure that the network where training takes place is secure. - - :string file_name: Name of Unity environment binary. - :int base_port: Baseline port number to connect to Unity environment over. worker_id increments over this. - If no environment is specified (i.e. file_name is None), the DEFAULT_EDITOR_PORT will be used. - :int worker_id: Offset from base_port. Used for training multiple environments simultaneously. - :bool no_graphics: Whether to run the Unity simulator in no-graphics mode - :int timeout_wait: Time (in seconds) to wait for connection from environment. - :list args: Addition Unity command line arguments - :list side_channels: Additional side channel for no-rl communication with Unity - :str log_folder: Optional folder to write the Unity Player log file into. Requires absolute path. - """ - atexit.register(self._close) - self._additional_args = additional_args or [] - self._no_graphics = no_graphics - # If base port is not specified, use BASE_ENVIRONMENT_PORT if we have - # an environment, otherwise DEFAULT_EDITOR_PORT - if base_port is None: - base_port = ( - self.BASE_ENVIRONMENT_PORT if file_name else self.DEFAULT_EDITOR_PORT - ) - self._port = base_port + worker_id - self._buffer_size = 12000 - # If true, this means the environment was successfully loaded - self._loaded = False - # The process that is started. If None, no process was started - self._process: Optional[subprocess.Popen] = None - self._timeout_wait: int = timeout_wait - self._communicator = self._get_communicator(worker_id, base_port, timeout_wait) - self._worker_id = worker_id - if side_channels is None: - side_channels = [] - default_training_side_channel: Optional[ - DefaultTrainingAnalyticsSideChannel - ] = None - if DefaultTrainingAnalyticsSideChannel.CHANNEL_ID not in [ - _.channel_id for _ in side_channels - ]: - default_training_side_channel = DefaultTrainingAnalyticsSideChannel() - side_channels.append(default_training_side_channel) - self._side_channel_manager = SideChannelManager(side_channels) - self._log_folder = log_folder - self.academy_capabilities: UnityRLCapabilitiesProto = None # type: ignore - - # If the environment name is None, a new environment will not be launched - # and the communicator will directly try to connect to an existing unity environment. - # If the worker-id is not 0 and the environment name is None, an error is thrown - if file_name is None and worker_id != 0: - raise UnityEnvironmentException( - "If the environment name is None, " - "the worker-id must be 0 in order to connect with the Editor." - ) - if file_name is not None: - try: - self._process = env_utils.launch_executable( - file_name, self._executable_args() - ) - except UnityEnvironmentException: - self._close(0) - raise - else: - logger.info( - f"Listening on port {self._port}. " - f"Start training by pressing the Play button in the Unity Editor." - ) - self._loaded = True - - rl_init_parameters_in = UnityRLInitializationInputProto( - seed=seed, - communication_version=self.API_VERSION, - package_version=mlagents_envs.__version__, - capabilities=UnityEnvironment._get_capabilities_proto(), - num_areas=num_areas, - ) - try: - aca_output = self._send_academy_parameters(rl_init_parameters_in) - aca_params = aca_output.rl_initialization_output - except UnityTimeOutException: - self._close(0) - raise - - if not UnityEnvironment._check_communication_compatibility( - aca_params.communication_version, - UnityEnvironment.API_VERSION, - aca_params.package_version, - ): - self._close(0) - UnityEnvironment._raise_version_exception(aca_params.communication_version) - - UnityEnvironment._warn_csharp_base_capabilities( - aca_params.capabilities, - aca_params.package_version, - UnityEnvironment.API_VERSION, - ) - - self._env_state: Dict[str, Tuple[DecisionSteps, TerminalSteps]] = {} - self._env_specs: Dict[str, BehaviorSpec] = {} - self._env_actions: Dict[str, ActionTuple] = {} - self._is_first_message = True - self._update_behavior_specs(aca_output) - self.academy_capabilities = aca_params.capabilities - if default_training_side_channel is not None: - default_training_side_channel.environment_initialized() - - @staticmethod - def _get_communicator(worker_id, base_port, timeout_wait): - return RpcCommunicator(worker_id, base_port, timeout_wait) - - def _executable_args(self) -> List[str]: - args: List[str] = [] - if self._no_graphics: - args += ["-nographics", "-batchmode"] - args += [UnityEnvironment._PORT_COMMAND_LINE_ARG, str(self._port)] - - # If the logfile arg isn't already set in the env args, - # try to set it to an output directory - logfile_set = "-logfile" in (arg.lower() for arg in self._additional_args) - if self._log_folder and not logfile_set: - log_file_path = os.path.join( - self._log_folder, f"Player-{self._worker_id}.log" - ) - args += ["-logFile", log_file_path] - # Add in arguments passed explicitly by the user. - args += self._additional_args - return args - - def _update_behavior_specs(self, output: UnityOutputProto) -> None: - init_output = output.rl_initialization_output - for brain_param in init_output.brain_parameters: - # Each BrainParameter in the rl_initialization_output should have at least one AgentInfo - # Get that agent, because we need some of its observations. - agent_infos = output.rl_output.agentInfos[brain_param.brain_name] - if agent_infos.value: - agent = agent_infos.value[0] - new_spec = behavior_spec_from_proto(brain_param, agent) - self._env_specs[brain_param.brain_name] = new_spec - logger.info(f"Connected new brain: {brain_param.brain_name}") - - def _update_state(self, output: UnityRLOutputProto) -> None: - """ - Collects experience information from all external brains in environment at current step. - """ - for brain_name in self._env_specs.keys(): - if brain_name in output.agentInfos: - agent_info_list = output.agentInfos[brain_name].value - self._env_state[brain_name] = steps_from_proto( - agent_info_list, self._env_specs[brain_name] - ) - else: - self._env_state[brain_name] = ( - DecisionSteps.empty(self._env_specs[brain_name]), - TerminalSteps.empty(self._env_specs[brain_name]), - ) - self._side_channel_manager.process_side_channel_message(output.side_channel) - - def reset(self) -> None: - if self._loaded: - outputs = self._communicator.exchange( - self._generate_reset_input(), self._poll_process - ) - if outputs is None: - raise UnityCommunicatorStoppedException("Communicator has exited.") - self._update_behavior_specs(outputs) - rl_output = outputs.rl_output - self._update_state(rl_output) - self._is_first_message = False - self._env_actions.clear() - else: - raise UnityEnvironmentException("No Unity environment is loaded.") - - @timed - def step(self) -> None: - if self._is_first_message: - return self.reset() - if not self._loaded: - raise UnityEnvironmentException("No Unity environment is loaded.") - # fill the blanks for missing actions - for group_name in self._env_specs: - if group_name not in self._env_actions: - n_agents = 0 - if group_name in self._env_state: - n_agents = len(self._env_state[group_name][0]) - self._env_actions[group_name] = self._env_specs[ - group_name - ].action_spec.empty_action(n_agents) - step_input = self._generate_step_input(self._env_actions) - with hierarchical_timer("communicator.exchange"): - outputs = self._communicator.exchange(step_input, self._poll_process) - if outputs is None: - raise UnityCommunicatorStoppedException("Communicator has exited.") - self._update_behavior_specs(outputs) - rl_output = outputs.rl_output - self._update_state(rl_output) - self._env_actions.clear() - - @property - def behavior_specs(self) -> MappingType[str, BehaviorSpec]: - return BehaviorMapping(self._env_specs) - - def _assert_behavior_exists(self, behavior_name: str) -> None: - if behavior_name not in self._env_specs: - raise UnityActionException( - f"The group {behavior_name} does not correspond to an existing " - f"agent group in the environment" - ) - - def set_actions(self, behavior_name: BehaviorName, action: ActionTuple) -> None: - self._assert_behavior_exists(behavior_name) - if behavior_name not in self._env_state: - return - action_spec = self._env_specs[behavior_name].action_spec - num_agents = len(self._env_state[behavior_name][0]) - action = action_spec._validate_action(action, num_agents, behavior_name) - self._env_actions[behavior_name] = action - - def set_action_for_agent( - self, behavior_name: BehaviorName, agent_id: AgentId, action: ActionTuple - ) -> None: - self._assert_behavior_exists(behavior_name) - if behavior_name not in self._env_state: - return - action_spec = self._env_specs[behavior_name].action_spec - action = action_spec._validate_action(action, 1, behavior_name) - if behavior_name not in self._env_actions: - num_agents = len(self._env_state[behavior_name][0]) - self._env_actions[behavior_name] = action_spec.empty_action(num_agents) - try: - index = np.where(self._env_state[behavior_name][0].agent_id == agent_id)[0][ - 0 - ] - except IndexError as ie: - raise IndexError( - "agent_id {} is did not request a decision at the previous step".format( - agent_id - ) - ) from ie - if action_spec.continuous_size > 0: - self._env_actions[behavior_name].continuous[index] = action.continuous[0, :] - if action_spec.discrete_size > 0: - self._env_actions[behavior_name].discrete[index] = action.discrete[0, :] - - def get_steps( - self, behavior_name: BehaviorName - ) -> Tuple[DecisionSteps, TerminalSteps]: - self._assert_behavior_exists(behavior_name) - return self._env_state[behavior_name] - - def _poll_process(self) -> None: - """ - Check the status of the subprocess. If it has exited, raise a UnityEnvironmentException - :return: None - """ - if not self._process: - return - poll_res = self._process.poll() - if poll_res is not None: - exc_msg = self._returncode_to_env_message(self._process.returncode) - raise UnityEnvironmentException(exc_msg) - - def close(self): - """ - Sends a shutdown signal to the unity environment, and closes the socket connection. - """ - if self._loaded: - self._close() - else: - raise UnityEnvironmentException("No Unity environment is loaded.") - - def _close(self, timeout: Optional[int] = None) -> None: - """ - Close the communicator and environment subprocess (if necessary). - - :int timeout: [Optional] Number of seconds to wait for the environment to shut down before - force-killing it. Defaults to `self.timeout_wait`. - """ - if timeout is None: - timeout = self._timeout_wait - self._loaded = False - self._communicator.close() - if self._process is not None: - # Wait a bit for the process to shutdown, but kill it if it takes too long - try: - self._process.wait(timeout=timeout) - logger.debug(self._returncode_to_env_message(self._process.returncode)) - except subprocess.TimeoutExpired: - logger.warning("Environment timed out shutting down. Killing...") - self._process.kill() - # Set to None so we don't try to close multiple times. - self._process = None - - @timed - def _generate_step_input( - self, vector_action: Dict[str, ActionTuple] - ) -> UnityInputProto: - rl_in = UnityRLInputProto() - for b in vector_action: - n_agents = len(self._env_state[b][0]) - if n_agents == 0: - continue - for i in range(n_agents): - action = AgentActionProto() - if vector_action[b].continuous is not None: - action.vector_actions_deprecated.extend( - vector_action[b].continuous[i] - ) - action.continuous_actions.extend(vector_action[b].continuous[i]) - if vector_action[b].discrete is not None: - action.vector_actions_deprecated.extend( - vector_action[b].discrete[i] - ) - action.discrete_actions.extend(vector_action[b].discrete[i]) - rl_in.agent_actions[b].value.extend([action]) - rl_in.command = STEP - rl_in.side_channel = bytes( - self._side_channel_manager.generate_side_channel_messages() - ) - return self._wrap_unity_input(rl_in) - - def _generate_reset_input(self) -> UnityInputProto: - rl_in = UnityRLInputProto() - rl_in.command = RESET - rl_in.side_channel = bytes( - self._side_channel_manager.generate_side_channel_messages() - ) - return self._wrap_unity_input(rl_in) - - def _send_academy_parameters( - self, init_parameters: UnityRLInitializationInputProto - ) -> UnityOutputProto: - inputs = UnityInputProto() - inputs.rl_initialization_input.CopyFrom(init_parameters) - return self._communicator.initialize(inputs, self._poll_process) - - @staticmethod - def _wrap_unity_input(rl_input: UnityRLInputProto) -> UnityInputProto: - result = UnityInputProto() - result.rl_input.CopyFrom(rl_input) - return result - - @staticmethod - def _returncode_to_signal_name(returncode: int) -> Optional[str]: - """ - Try to convert return codes into their corresponding signal name. - E.g. returncode_to_signal_name(-2) -> "SIGINT" - """ - try: - # A negative value -N indicates that the child was terminated by signal N (POSIX only). - s = signal.Signals(-returncode) - return s.name - except Exception: - # Should generally be a ValueError, but catch everything just in case. - return None - - @staticmethod - def _returncode_to_env_message(returncode: int) -> str: - signal_name = UnityEnvironment._returncode_to_signal_name(returncode) - signal_name = f" ({signal_name})" if signal_name else "" - return f"Environment shut down with return code {returncode}{signal_name}." diff --git a/3DOrangeTree/mlagents_envs/exception.py b/3DOrangeTree/mlagents_envs/exception.py deleted file mode 100644 index 324cdd0..0000000 --- a/3DOrangeTree/mlagents_envs/exception.py +++ /dev/null @@ -1,86 +0,0 @@ -class UnityException(Exception): - """ - Any error related to ml-agents environment. - """ - - pass - - -class UnityEnvironmentException(UnityException): - """ - Related to errors starting and closing environment. - """ - - pass - - -class UnityCommunicationException(UnityException): - """ - Related to errors with the communicator. - """ - - pass - - -class UnityCommunicatorStoppedException(UnityException): - """ - Raised when communicator has stopped gracefully. - """ - - pass - - -class UnityObservationException(UnityException): - """ - Related to errors with receiving observations. - """ - - pass - - -class UnityActionException(UnityException): - """ - Related to errors with sending actions. - """ - - pass - - -class UnityTimeOutException(UnityException): - """ - Related to errors with communication timeouts. - """ - - pass - - -class UnitySideChannelException(UnityException): - """ - Related to errors with side channels. - """ - - pass - - -class UnityWorkerInUseException(UnityException): - """ - This error occurs when the port for a certain worker ID is already reserved. - """ - - MESSAGE_TEMPLATE = ( - "Couldn't start socket communication because worker number {} is still in use. " - "You may need to manually close a previously opened environment " - "or use a different worker number." - ) - - def __init__(self, worker_id): - message = self.MESSAGE_TEMPLATE.format(str(worker_id)) - super().__init__(message) - - -class UnityPolicyException(UnityException): - """ - Related to errors with the Trainer. - """ - - pass diff --git a/3DOrangeTree/mlagents_envs/logging_util.py b/3DOrangeTree/mlagents_envs/logging_util.py deleted file mode 100644 index ddd4e3b..0000000 --- a/3DOrangeTree/mlagents_envs/logging_util.py +++ /dev/null @@ -1,63 +0,0 @@ -import logging # noqa I251 -import sys - -CRITICAL = logging.CRITICAL -FATAL = logging.FATAL -ERROR = logging.ERROR -WARNING = logging.WARNING -INFO = logging.INFO -DEBUG = logging.DEBUG -NOTSET = logging.NOTSET - -_loggers = set() -_log_level = NOTSET -DATE_FORMAT = "%Y-%m-%d %H:%M:%S" -DEBUG_LOG_FORMAT = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" -LOG_FORMAT = "[%(levelname)s] %(message)s" - - -def get_logger(name: str) -> logging.Logger: - """ - Create a logger with the specified name. The logger will use the log level - specified by set_log_level() - """ - logger = logging.getLogger(name=name) - - if _log_level == DEBUG: - formatter = logging.Formatter(fmt=DEBUG_LOG_FORMAT, datefmt=DATE_FORMAT) - else: - formatter = logging.Formatter(fmt=LOG_FORMAT) - handler = logging.StreamHandler(stream=sys.stdout) - handler.setFormatter(formatter) - logger.addHandler(handler) - - # If we've already set the log level, make sure new loggers use it - if _log_level != NOTSET: - logger.setLevel(_log_level) - - # Keep track of this logger so that we can change the log level later - _loggers.add(logger) - return logger - - -def set_log_level(log_level: int) -> None: - """ - Set the ML-Agents logging level. This will also configure the logging format (if it hasn't already been set). - """ - global _log_level - _log_level = log_level - - for logger in _loggers: - logger.setLevel(log_level) - - if log_level == DEBUG: - formatter = logging.Formatter(fmt=DEBUG_LOG_FORMAT, datefmt=DATE_FORMAT) - else: - formatter = logging.Formatter(LOG_FORMAT) - _set_formatter_for_all_loggers(formatter) - - -def _set_formatter_for_all_loggers(formatter: logging.Formatter) -> None: - for logger in _loggers: - for handler in logger.handlers[:]: - handler.setFormatter(formatter) diff --git a/3DOrangeTree/mlagents_envs/mock_communicator.py b/3DOrangeTree/mlagents_envs/mock_communicator.py deleted file mode 100644 index 0e425e2..0000000 --- a/3DOrangeTree/mlagents_envs/mock_communicator.py +++ /dev/null @@ -1,111 +0,0 @@ -from typing import Optional - -from .communicator import Communicator, PollCallback -from .environment import UnityEnvironment -from mlagents_envs.communicator_objects.unity_rl_output_pb2 import UnityRLOutputProto -from mlagents_envs.communicator_objects.brain_parameters_pb2 import ( - BrainParametersProto, - ActionSpecProto, -) -from mlagents_envs.communicator_objects.unity_rl_initialization_output_pb2 import ( - UnityRLInitializationOutputProto, -) -from mlagents_envs.communicator_objects.unity_input_pb2 import UnityInputProto -from mlagents_envs.communicator_objects.unity_output_pb2 import UnityOutputProto -from mlagents_envs.communicator_objects.agent_info_pb2 import AgentInfoProto -from mlagents_envs.communicator_objects.observation_pb2 import ( - ObservationProto, - NONE as COMPRESSION_TYPE_NONE, - PNG as COMPRESSION_TYPE_PNG, -) - - -class MockCommunicator(Communicator): - def __init__( - self, - discrete_action=False, - visual_inputs=0, - num_agents=3, - brain_name="RealFakeBrain", - vec_obs_size=3, - ): - """ - Python side of the grpc communication. Python is the client and Unity the server - """ - super().__init__() - self.is_discrete = discrete_action - self.steps = 0 - self.visual_inputs = visual_inputs - self.has_been_closed = False - self.num_agents = num_agents - self.brain_name = brain_name - self.vec_obs_size = vec_obs_size - - def initialize( - self, inputs: UnityInputProto, poll_callback: Optional[PollCallback] = None - ) -> UnityOutputProto: - if self.is_discrete: - action_spec = ActionSpecProto( - num_discrete_actions=2, discrete_branch_sizes=[3, 2] - ) - else: - action_spec = ActionSpecProto(num_continuous_actions=2) - bp = BrainParametersProto( - brain_name=self.brain_name, is_training=True, action_spec=action_spec - ) - rl_init = UnityRLInitializationOutputProto( - name="RealFakeAcademy", - communication_version=UnityEnvironment.API_VERSION, - package_version="mock_package_version", - log_path="", - brain_parameters=[bp], - ) - output = UnityRLOutputProto(agentInfos=self._get_agent_infos()) - return UnityOutputProto(rl_initialization_output=rl_init, rl_output=output) - - def _get_agent_infos(self): - dict_agent_info = {} - list_agent_info = [] - vector_obs = [1, 2, 3] - - observations = [ - ObservationProto( - compressed_data=None, - shape=[30, 40, 3], - compression_type=COMPRESSION_TYPE_PNG, - ) - for _ in range(self.visual_inputs) - ] - vector_obs_proto = ObservationProto( - float_data=ObservationProto.FloatData(data=vector_obs), - shape=[len(vector_obs)], - compression_type=COMPRESSION_TYPE_NONE, - ) - observations.append(vector_obs_proto) - - for i in range(self.num_agents): - list_agent_info.append( - AgentInfoProto( - reward=1, - done=(i == 2), - max_step_reached=False, - id=i, - observations=observations, - ) - ) - dict_agent_info["RealFakeBrain"] = UnityRLOutputProto.ListAgentInfoProto( - value=list_agent_info - ) - return dict_agent_info - - def exchange( - self, inputs: UnityInputProto, poll_callback: Optional[PollCallback] = None - ) -> UnityOutputProto: - result = UnityRLOutputProto(agentInfos=self._get_agent_infos()) - return UnityOutputProto(rl_output=result) - - def close(self): - """ - Sends a shutdown signal to the unity environment, and closes the grpc connection. - """ - self.has_been_closed = True diff --git a/3DOrangeTree/mlagents_envs/registry/__init__.py b/3DOrangeTree/mlagents_envs/registry/__init__.py deleted file mode 100644 index 74c0e8a..0000000 --- a/3DOrangeTree/mlagents_envs/registry/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from mlagents_envs.registry.unity_env_registry import ( # noqa F401 - default_registry, - UnityEnvRegistry, -) diff --git a/3DOrangeTree/mlagents_envs/registry/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/mlagents_envs/registry/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index 14a70bd..0000000 Binary files a/3DOrangeTree/mlagents_envs/registry/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/registry/__pycache__/base_registry_entry.cpython-37.pyc b/3DOrangeTree/mlagents_envs/registry/__pycache__/base_registry_entry.cpython-37.pyc deleted file mode 100644 index 7cefcab..0000000 Binary files a/3DOrangeTree/mlagents_envs/registry/__pycache__/base_registry_entry.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/registry/__pycache__/binary_utils.cpython-37.pyc b/3DOrangeTree/mlagents_envs/registry/__pycache__/binary_utils.cpython-37.pyc deleted file mode 100644 index 473ef40..0000000 Binary files a/3DOrangeTree/mlagents_envs/registry/__pycache__/binary_utils.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/registry/__pycache__/remote_registry_entry.cpython-37.pyc b/3DOrangeTree/mlagents_envs/registry/__pycache__/remote_registry_entry.cpython-37.pyc deleted file mode 100644 index 0e9a36e..0000000 Binary files a/3DOrangeTree/mlagents_envs/registry/__pycache__/remote_registry_entry.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/registry/__pycache__/unity_env_registry.cpython-37.pyc b/3DOrangeTree/mlagents_envs/registry/__pycache__/unity_env_registry.cpython-37.pyc deleted file mode 100644 index a773de7..0000000 Binary files a/3DOrangeTree/mlagents_envs/registry/__pycache__/unity_env_registry.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/registry/base_registry_entry.py b/3DOrangeTree/mlagents_envs/registry/base_registry_entry.py deleted file mode 100644 index f72009f..0000000 --- a/3DOrangeTree/mlagents_envs/registry/base_registry_entry.py +++ /dev/null @@ -1,56 +0,0 @@ -from abc import abstractmethod -from typing import Any, Optional -from mlagents_envs.base_env import BaseEnv - - -class BaseRegistryEntry: - def __init__( - self, - identifier: str, - expected_reward: Optional[float], - description: Optional[str], - ): - """ - BaseRegistryEntry allows launching a Unity Environment with its make method. - :param identifier: The name of the Unity Environment. - :param expected_reward: The cumulative reward that an Agent must receive - for the task to be considered solved. - :param description: A description of the Unity Environment. Contains human - readable information about potential special arguments that the make method can - take as well as information regarding the observation, reward, actions, - behaviors and number of agents in the Environment. - """ - self._identifier = identifier - self._expected_reward = expected_reward - self._description = description - - @property - def identifier(self) -> str: - """ - The unique identifier of the entry - """ - return self._identifier - - @property - def expected_reward(self) -> Optional[float]: - """ - The cumulative reward that an Agent must receive for the task to be considered - solved. - """ - return self._expected_reward - - @property - def description(self) -> Optional[str]: - """ - A description of the Unity Environment the entry can make. - """ - return self._description - - @abstractmethod - def make(self, **kwargs: Any) -> BaseEnv: - """ - This method creates a Unity BaseEnv (usually a UnityEnvironment). - """ - raise NotImplementedError( - f"The make() method not implemented for entry {self.identifier}" - ) diff --git a/3DOrangeTree/mlagents_envs/registry/binary_utils.py b/3DOrangeTree/mlagents_envs/registry/binary_utils.py deleted file mode 100644 index c75d8dd..0000000 --- a/3DOrangeTree/mlagents_envs/registry/binary_utils.py +++ /dev/null @@ -1,235 +0,0 @@ -import urllib.request -import tempfile -import os -import uuid -import shutil -import glob -import yaml -import hashlib - -from zipfile import ZipFile -from sys import platform -from typing import Tuple, Optional, Dict, Any - -from mlagents_envs.env_utils import validate_environment_path - -from mlagents_envs.logging_util import get_logger - -logger = get_logger(__name__) - -# The default logical block size is 8192 bytes (8 KB) for UFS file systems. -BLOCK_SIZE = 8192 - - -def get_local_binary_path(name: str, url: str) -> str: - """ - Returns the path to the executable previously downloaded with the name argument. If - None is found, the executable at the url argument will be downloaded and stored - under name for future uses. - :param name: The name that will be given to the folder containing the extracted data - :param url: The URL of the zip file - """ - NUMBER_ATTEMPTS = 5 - path = get_local_binary_path_if_exists(name, url) - if path is None: - logger.debug( - f"Local environment {name} not found, downloading environment from {url}" - ) - for attempt in range(NUMBER_ATTEMPTS): # Perform 5 attempts at downloading the file - if path is not None: - break - try: - download_and_extract_zip(url, name) - except Exception: - if attempt + 1 < NUMBER_ATTEMPTS: - logger.warning( - f"Attempt {attempt + 1} / {NUMBER_ATTEMPTS}" - ": Failed to download and extract binary." - ) - else: - raise - path = get_local_binary_path_if_exists(name, url) - - if path is None: - raise FileNotFoundError( - f"Binary not found, make sure {url} is a valid url to " - "a zip folder containing a valid Unity executable" - ) - return path - - -def get_local_binary_path_if_exists(name: str, url: str) -> Optional[str]: - """ - Recursively searches for a Unity executable in the extracted files folders. This is - platform dependent : It will only return a Unity executable compatible with the - computer's OS. If no executable is found, None will be returned. - :param name: The name/identifier of the executable - :param url: The url the executable was downloaded from (for verification) - """ - _, bin_dir = get_tmp_dir() - extension = None - - if platform == "linux" or platform == "linux2": - extension = "*.x86_64" - if platform == "darwin": - extension = "*.app" - if platform == "win32": - extension = "*.exe" - if extension is None: - raise NotImplementedError("No extensions found for this platform.") - url_hash = "-" + hashlib.md5(url.encode()).hexdigest() - path = os.path.join(bin_dir, name + url_hash, "**", extension) - candidates = glob.glob(path, recursive=True) - if len(candidates) == 0: - return None - else: - for c in candidates: - # Unity sometimes produces another .exe file that we must filter out - if "UnityCrashHandler64" not in c: - # If the file is not valid, return None and delete faulty directory - if validate_environment_path(c) is None: - shutil.rmtree(c) - return None - return c - return None - - -def get_tmp_dir() -> Tuple[str, str]: - """ - Returns the path to the folder containing the downloaded zip files and the extracted - binaries. If these folders do not exist, they will be created. - :retrun: Tuple containing path to : (zip folder, extracted files folder) - """ - TEMPDIR = "/tmp" if platform == "darwin" else tempfile.gettempdir() - MLAGENTS = "ml-agents-binaries" - TMP_FOLDER_NAME = "tmp" - BINARY_FOLDER_NAME = "binaries" - mla_directory = os.path.join(TEMPDIR, MLAGENTS) - if not os.path.exists(mla_directory): - os.makedirs(mla_directory) - os.chmod(mla_directory, 16877) - zip_directory = os.path.join(TEMPDIR, MLAGENTS, TMP_FOLDER_NAME) - if not os.path.exists(zip_directory): - os.makedirs(zip_directory) - os.chmod(zip_directory, 16877) - bin_directory = os.path.join(TEMPDIR, MLAGENTS, BINARY_FOLDER_NAME) - if not os.path.exists(bin_directory): - os.makedirs(bin_directory) - os.chmod(bin_directory, 16877) - return (zip_directory, bin_directory) - - -def download_and_extract_zip(url: str, name: str) -> None: - """ - Downloads a zip file under a URL, extracts its contents into a folder with the name - argument and gives chmod 755 to all the files it contains. Files are downloaded and - extracted into special folders in the temp folder of the machine. - :param url: The URL of the zip file - :param name: The name that will be given to the folder containing the extracted data - """ - zip_dir, bin_dir = get_tmp_dir() - url_hash = "-" + hashlib.md5(url.encode()).hexdigest() - binary_path = os.path.join(bin_dir, name + url_hash) - if os.path.exists(binary_path): - shutil.rmtree(binary_path) - - # Download zip - try: - request = urllib.request.urlopen(url, timeout=30) - except urllib.error.HTTPError as e: # type: ignore - e.msg += " " + url - raise - zip_size = int(request.headers["content-length"]) - zip_file_path = os.path.join(zip_dir, str(uuid.uuid4()) + ".zip") - with open(zip_file_path, "wb") as zip_file: - downloaded = 0 - while True: - buffer = request.read(BLOCK_SIZE) - if not buffer: - # There is nothing more to read - break - downloaded += len(buffer) - zip_file.write(buffer) - downloaded_percent = downloaded / zip_size * 100 - print_progress(f" Downloading {name}", downloaded_percent) - print("") - - # Extraction - with ZipFileWithProgress(zip_file_path, "r") as zip_ref: - zip_ref.extract_zip(f" Extracting {name}", binary_path) # type: ignore - print("") - - # Clean up zip - print_progress(f" Cleaning up {name}", 0) - os.remove(zip_file_path) - - # Give permission - for f in glob.glob(binary_path + "/**/*", recursive=True): - # 16877 is octal 40755, which denotes a directory with permissions 755 - os.chmod(f, 16877) - print_progress(f" Cleaning up {name}", 100) - print("") - - -def print_progress(prefix: str, percent: float) -> None: - """ - Displays a single progress bar in the terminal with value percent. - :param prefix: The string that will precede the progress bar. - :param percent: The percent progression of the bar (min is 0, max is 100) - """ - BAR_LEN = 20 - percent = min(100, max(0, percent)) - bar_progress = min(int(percent / 100 * BAR_LEN), BAR_LEN) - bar = "|" + "\u2588" * bar_progress + " " * (BAR_LEN - bar_progress) + "|" - str_percent = "%3.0f%%" % percent - print(f"{prefix} : {bar} {str_percent} \r", end="", flush=True) - - -def load_remote_manifest(url: str) -> Dict[str, Any]: - """ - Converts a remote yaml file into a Python dictionary - """ - tmp_dir, _ = get_tmp_dir() - try: - request = urllib.request.urlopen(url, timeout=30) - except urllib.error.HTTPError as e: # type: ignore - e.msg += " " + url - raise - manifest_path = os.path.join(tmp_dir, str(uuid.uuid4()) + ".yaml") - with open(manifest_path, "wb") as manifest: - while True: - buffer = request.read(BLOCK_SIZE) - if not buffer: - # There is nothing more to read - break - manifest.write(buffer) - try: - result = load_local_manifest(manifest_path) - finally: - os.remove(manifest_path) - return result - - -def load_local_manifest(path: str) -> Dict[str, Any]: - """ - Converts a local yaml file into a Python dictionary - """ - with open(path) as data_file: - return yaml.safe_load(data_file) - - -class ZipFileWithProgress(ZipFile): - """ - This is a helper class inheriting from ZipFile that allows to display a progress - bar while the files are being extracted. - """ - - def extract_zip(self, prefix: str, path: str) -> None: - members = self.namelist() - path = os.fspath(path) - total = len(members) - n = 0 - for zipinfo in members: - self.extract(zipinfo, path, None) # type: ignore - n += 1 - print_progress(prefix, n / total * 100) diff --git a/3DOrangeTree/mlagents_envs/registry/remote_registry_entry.py b/3DOrangeTree/mlagents_envs/registry/remote_registry_entry.py deleted file mode 100644 index c267f7d..0000000 --- a/3DOrangeTree/mlagents_envs/registry/remote_registry_entry.py +++ /dev/null @@ -1,71 +0,0 @@ -from sys import platform -from typing import Optional, Any, List -from mlagents_envs.environment import UnityEnvironment -from mlagents_envs.base_env import BaseEnv -from mlagents_envs.registry.binary_utils import get_local_binary_path -from mlagents_envs.registry.base_registry_entry import BaseRegistryEntry - - -class RemoteRegistryEntry(BaseRegistryEntry): - def __init__( - self, - identifier: str, - expected_reward: Optional[float], - description: Optional[str], - linux_url: Optional[str], - darwin_url: Optional[str], - win_url: Optional[str], - additional_args: Optional[List[str]] = None, - ): - """ - A RemoteRegistryEntry is an implementation of BaseRegistryEntry that uses a - Unity executable downloaded from the internet to launch a UnityEnvironment. - __Note__: The url provided must be a link to a `.zip` file containing a single - compressed folder with the executable inside. There can only be one executable - in the folder and it must be at the root of the folder. - :param identifier: The name of the Unity Environment. - :param expected_reward: The cumulative reward that an Agent must receive - for the task to be considered solved. - :param description: A description of the Unity Environment. Contains human - readable information about potential special arguments that the make method can - take as well as information regarding the observation, reward, actions, - behaviors and number of agents in the Environment. - :param linux_url: The url of the Unity executable for the Linux platform - :param darwin_url: The url of the Unity executable for the OSX platform - :param win_url: The url of the Unity executable for the Windows platform - """ - super().__init__(identifier, expected_reward, description) - self._linux_url = linux_url - self._darwin_url = darwin_url - self._win_url = win_url - self._add_args = additional_args - - def make(self, **kwargs: Any) -> BaseEnv: - """ - Returns the UnityEnvironment that corresponds to the Unity executable found at - the provided url. The arguments passed to this method will be passed to the - constructor of the UnityEnvironment (except for the file_name argument) - """ - url = None - if platform == "linux" or platform == "linux2": - url = self._linux_url - if platform == "darwin": - url = self._darwin_url - if platform == "win32": - url = self._win_url - if url is None: - raise FileNotFoundError( - f"The entry {self.identifier} does not contain a valid url for this " - "platform" - ) - path = get_local_binary_path(self.identifier, url) - if "file_name" in kwargs: - kwargs.pop("file_name") - args: List[str] = [] - if "additional_args" in kwargs: - if kwargs["additional_args"] is not None: - args += kwargs["additional_args"] - if self._add_args is not None: - args += self._add_args - kwargs["additional_args"] = args - return UnityEnvironment(file_name=path, **kwargs) diff --git a/3DOrangeTree/mlagents_envs/registry/unity_env_registry.py b/3DOrangeTree/mlagents_envs/registry/unity_env_registry.py deleted file mode 100644 index 639f857..0000000 --- a/3DOrangeTree/mlagents_envs/registry/unity_env_registry.py +++ /dev/null @@ -1,125 +0,0 @@ -from typing import Dict, Iterator, Any, List -from collections.abc import Mapping -from mlagents_envs.registry.base_registry_entry import BaseRegistryEntry -from mlagents_envs.registry.binary_utils import ( - load_local_manifest, - load_remote_manifest, -) -from mlagents_envs.registry.remote_registry_entry import RemoteRegistryEntry - - -class UnityEnvRegistry(Mapping): - """ - ### UnityEnvRegistry - Provides a library of Unity environments that can be launched without the need - of downloading the Unity Editor. - The UnityEnvRegistry implements a Map, to access an entry of the Registry, use: - ```python - registry = UnityEnvRegistry() - entry = registry[] - ``` - An entry has the following properties : - * `identifier` : Uniquely identifies this environment - * `expected_reward` : Corresponds to the reward an agent must obtained for the task - to be considered completed. - * `description` : A human readable description of the environment. - - To launch a Unity environment from a registry entry, use the `make` method: - ```python - registry = UnityEnvRegistry() - env = registry[].make() - ``` - """ - - def __init__(self): - self._REGISTERED_ENVS: Dict[str, BaseRegistryEntry] = {} - self._manifests: List[str] = [] - self._sync = True - - def register(self, new_entry: BaseRegistryEntry) -> None: - """ - Registers a new BaseRegistryEntry to the registry. The - BaseRegistryEntry.identifier value will be used as indexing key. - If two are more environments are registered under the same key, the most - recentry added will replace the others. - """ - self._REGISTERED_ENVS[new_entry.identifier] = new_entry - - def register_from_yaml(self, path_to_yaml: str) -> None: - """ - Registers the environments listed in a yaml file (either local or remote). Note - that the entries are registered lazily: the registration will only happen when - an environment is accessed. - The yaml file must have the following format : - ```yaml - environments: - - : - expected_reward: - description: | - - linux_url: - darwin_url: - win_url: - - - : - expected_reward: - description: | - - linux_url: - darwin_url: - win_url: - - - ... - ``` - :param path_to_yaml: A local path or url to the yaml file - """ - self._manifests.append(path_to_yaml) - self._sync = False - - def _load_all_manifests(self) -> None: - if not self._sync: - for path_to_yaml in self._manifests: - if path_to_yaml[:4] == "http": - manifest = load_remote_manifest(path_to_yaml) - else: - manifest = load_local_manifest(path_to_yaml) - for env in manifest["environments"]: - remote_entry_args = list(env.values())[0] - remote_entry_args["identifier"] = list(env.keys())[0] - self.register(RemoteRegistryEntry(**remote_entry_args)) - self._manifests = [] - self._sync = True - - def clear(self) -> None: - """ - Deletes all entries in the registry. - """ - self._REGISTERED_ENVS.clear() - self._manifests = [] - self._sync = True - - def __getitem__(self, identifier: str) -> BaseRegistryEntry: - """ - Returns the BaseRegistryEntry with the provided identifier. BaseRegistryEntry - can then be used to make a Unity Environment. - :param identifier: The identifier of the BaseRegistryEntry - :returns: The associated BaseRegistryEntry - """ - self._load_all_manifests() - if identifier not in self._REGISTERED_ENVS: - raise KeyError(f"The entry {identifier} is not present in the registry.") - return self._REGISTERED_ENVS[identifier] - - def __len__(self) -> int: - self._load_all_manifests() - return len(self._REGISTERED_ENVS) - - def __iter__(self) -> Iterator[Any]: - self._load_all_manifests() - yield from self._REGISTERED_ENVS - - -default_registry = UnityEnvRegistry() -default_registry.register_from_yaml( - "https://storage.googleapis.com/mlagents-test-environments/1.0.0/manifest.yaml" -) # noqa E501 diff --git a/3DOrangeTree/mlagents_envs/rpc_communicator.py b/3DOrangeTree/mlagents_envs/rpc_communicator.py deleted file mode 100644 index 7544800..0000000 --- a/3DOrangeTree/mlagents_envs/rpc_communicator.py +++ /dev/null @@ -1,158 +0,0 @@ -import grpc -from typing import Optional - -from multiprocessing import Pipe -from sys import platform -import socket -import time -from concurrent.futures import ThreadPoolExecutor - -from .communicator import Communicator, PollCallback -from mlagents_envs.communicator_objects.unity_to_external_pb2_grpc import ( - UnityToExternalProtoServicer, - add_UnityToExternalProtoServicer_to_server, -) -from mlagents_envs.communicator_objects.unity_message_pb2 import UnityMessageProto -from mlagents_envs.communicator_objects.unity_input_pb2 import UnityInputProto -from mlagents_envs.communicator_objects.unity_output_pb2 import UnityOutputProto -from .exception import UnityTimeOutException, UnityWorkerInUseException - - -class UnityToExternalServicerImplementation(UnityToExternalProtoServicer): - def __init__(self): - self.parent_conn, self.child_conn = Pipe() - - def Initialize(self, request, context): - self.child_conn.send(request) - return self.child_conn.recv() - - def Exchange(self, request, context): - self.child_conn.send(request) - return self.child_conn.recv() - - -class RpcCommunicator(Communicator): - def __init__(self, worker_id=0, base_port=5005, timeout_wait=30): - """ - Python side of the grpc communication. Python is the server and Unity the client - - - :int base_port: Baseline port number to connect to Unity environment over. worker_id increments over this. - :int worker_id: Offset from base_port. Used for training multiple environments simultaneously. - :int timeout_wait: Timeout (in seconds) to wait for a response before exiting. - """ - super().__init__(worker_id, base_port) - self.port = base_port + worker_id - self.worker_id = worker_id - self.timeout_wait = 9000000 #timeout_wait - self.server = None - self.unity_to_external = None - self.is_open = False - self.create_server() - - def create_server(self): - """ - Creates the GRPC server. - """ - self.check_port(self.port) - - try: - # Establish communication grpc - self.server = grpc.server( - thread_pool=ThreadPoolExecutor(max_workers=10), - options=(("grpc.so_reuseport", 1),), - ) - self.unity_to_external = UnityToExternalServicerImplementation() - add_UnityToExternalProtoServicer_to_server( - self.unity_to_external, self.server - ) - # Using unspecified address, which means that grpc is communicating on all IPs - # This is so that the docker container can connect. - self.server.add_insecure_port("[::]:" + str(self.port)) - self.server.start() - self.is_open = True - except Exception: - raise UnityWorkerInUseException(self.worker_id) - - def check_port(self, port): - """ - Attempts to bind to the requested communicator port, checking if it is already in use. - """ - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if platform == "linux" or platform == "linux2": - # On linux, the port remains unusable for TIME_WAIT=60 seconds after closing - # SO_REUSEADDR frees the port right after closing the environment - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - s.bind(("localhost", port)) - except OSError: - raise UnityWorkerInUseException(self.worker_id) - finally: - s.close() - - def poll_for_timeout(self, poll_callback: Optional[PollCallback] = None) -> None: - """ - Polls the GRPC parent connection for data, to be used before calling recv. This prevents - us from hanging indefinitely in the case where the environment process has died or was not - launched. - - Additionally, a callback can be passed to periodically check the state of the environment. - This is used to detect the case when the environment dies without cleaning up the connection, - so that we can stop sooner and raise a more appropriate error. - """ - deadline = time.monotonic() + self.timeout_wait - callback_timeout_wait = self.timeout_wait // 10 - while time.monotonic() < deadline: - if self.unity_to_external.parent_conn.poll(callback_timeout_wait): - # Got an acknowledgment from the connection - return - if poll_callback: - # Fire the callback - if it detects something wrong, it should raise an exception. - poll_callback() - - # Got this far without reading any data from the connection, so it must be dead. - raise UnityTimeOutException( - "The Unity environment took too long to respond. Make sure that :\n" - "\t The environment does not need user interaction to launch\n" - '\t The Agents\' Behavior Parameters > Behavior Type is set to "Default"\n' - "\t The environment and the Python interface have compatible versions.\n" - "\t If you're running on a headless server without graphics support, turn off display " - "by either passing --no-graphics option or build your Unity executable as server build." - ) - - def initialize( - self, inputs: UnityInputProto, poll_callback: Optional[PollCallback] = None - ) -> UnityOutputProto: - self.poll_for_timeout(poll_callback) - aca_param = self.unity_to_external.parent_conn.recv().unity_output - message = UnityMessageProto() - message.header.status = 200 - message.unity_input.CopyFrom(inputs) - self.unity_to_external.parent_conn.send(message) - self.unity_to_external.parent_conn.recv() - return aca_param - - def exchange( - self, inputs: UnityInputProto, poll_callback: Optional[PollCallback] = None - ) -> Optional[UnityOutputProto]: - message = UnityMessageProto() - message.header.status = 200 - message.unity_input.CopyFrom(inputs) - self.unity_to_external.parent_conn.send(message) - self.poll_for_timeout(poll_callback) - output = self.unity_to_external.parent_conn.recv() - if output.header.status != 200: - return None - return output.unity_output - - def close(self): - """ - Sends a shutdown signal to the unity environment, and closes the grpc connection. - """ - if self.is_open: - message_input = UnityMessageProto() - message_input.header.status = 400 - self.unity_to_external.parent_conn.send(message_input) - self.unity_to_external.parent_conn.close() - self.server.stop(False) - self.is_open = False diff --git a/3DOrangeTree/mlagents_envs/rpc_utils.py b/3DOrangeTree/mlagents_envs/rpc_utils.py deleted file mode 100644 index 5ea4db7..0000000 --- a/3DOrangeTree/mlagents_envs/rpc_utils.py +++ /dev/null @@ -1,431 +0,0 @@ -from mlagents_envs.base_env import ( - ActionSpec, - ObservationSpec, - DimensionProperty, - BehaviorSpec, - DecisionSteps, - TerminalSteps, - ObservationType, -) -from mlagents_envs.exception import UnityObservationException -from mlagents_envs.timers import hierarchical_timer, timed -from mlagents_envs.communicator_objects.agent_info_pb2 import AgentInfoProto -from mlagents_envs.communicator_objects.observation_pb2 import ( - ObservationProto, - NONE as COMPRESSION_TYPE_NONE, -) -from mlagents_envs.communicator_objects.brain_parameters_pb2 import BrainParametersProto -import numpy as np -import io -from typing import cast, List, Tuple, Collection, Optional, Iterable -from PIL import Image - - -PNG_HEADER = b"\x89PNG\r\n\x1a\n" - - -def behavior_spec_from_proto( - brain_param_proto: BrainParametersProto, agent_info: AgentInfoProto -) -> BehaviorSpec: - """ - Converts brain parameter and agent info proto to BehaviorSpec object. - :param brain_param_proto: protobuf object. - :param agent_info: protobuf object. - :return: BehaviorSpec object. - """ - observation_specs = [] - for obs in agent_info.observations: - observation_specs.append( - ObservationSpec( - name=obs.name, - shape=tuple(obs.shape), - observation_type=ObservationType(obs.observation_type), - dimension_property=tuple( - DimensionProperty(dim) for dim in obs.dimension_properties - ) - if len(obs.dimension_properties) > 0 - else (DimensionProperty.UNSPECIFIED,) * len(obs.shape), - ) - ) - - # proto from communicator < v1.3 does not set action spec, use deprecated fields instead - if ( - brain_param_proto.action_spec.num_continuous_actions == 0 - and brain_param_proto.action_spec.num_discrete_actions == 0 - ): - if brain_param_proto.vector_action_space_type_deprecated == 1: - action_spec = ActionSpec( - brain_param_proto.vector_action_size_deprecated[0], () - ) - else: - action_spec = ActionSpec( - 0, tuple(brain_param_proto.vector_action_size_deprecated) - ) - else: - action_spec_proto = brain_param_proto.action_spec - action_spec = ActionSpec( - action_spec_proto.num_continuous_actions, - tuple(branch for branch in action_spec_proto.discrete_branch_sizes), - ) - return BehaviorSpec(observation_specs, action_spec) - - -class OffsetBytesIO: - """ - Simple file-like class that wraps a bytes, and allows moving its "start" - position in the bytes. This is only used for reading concatenated PNGs, - because Pillow always calls seek(0) at the start of reading. - """ - - __slots__ = ["fp", "offset"] - - def __init__(self, data: bytes): - self.fp = io.BytesIO(data) - self.offset = 0 - - def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: - if whence == io.SEEK_SET: - res = self.fp.seek(offset + self.offset) - return res - self.offset - raise NotImplementedError() - - def tell(self) -> int: - return self.fp.tell() - self.offset - - def read(self, size: int = -1) -> bytes: - return self.fp.read(size) - - def original_tell(self) -> int: - """ - Returns the offset into the original byte array - """ - return self.fp.tell() - - -@timed -def process_pixels( - image_bytes: bytes, expected_channels: int, mappings: Optional[List[int]] = None -) -> np.ndarray: - """ - Converts byte array observation image into numpy array, re-sizes it, - and optionally converts it to grey scale - :param image_bytes: input byte array corresponding to image - :param expected_channels: Expected output channels - :return: processed numpy array of observation from environment - """ - image_fp = OffsetBytesIO(image_bytes) - - image_arrays = [] - # Read the images back from the bytes (without knowing the sizes). - while True: - with hierarchical_timer("image_decompress"): - image = Image.open(image_fp) - # Normally Image loads lazily, load() forces it to do loading in the timer scope. - image.load() - image_arrays.append(np.array(image, dtype=np.float32) / 255.0) - - # Look for the next header, starting from the current stream location - try: - new_offset = image_bytes.index(PNG_HEADER, image_fp.original_tell()) - image_fp.offset = new_offset - except ValueError: - # Didn't find the header, so must be at the end. - break - - if mappings is not None and len(mappings) > 0: - return _process_images_mapping(image_arrays, mappings) - else: - return _process_images_num_channels(image_arrays, expected_channels) - - -def _process_images_mapping(image_arrays, mappings): - """ - Helper function for processing decompressed images with compressed channel mappings. - """ - image_arrays = np.concatenate(image_arrays, axis=2).transpose((2, 0, 1)) - - if len(mappings) != len(image_arrays): - raise UnityObservationException( - f"Compressed observation and its mapping had different number of channels - " - f"observation had {len(image_arrays)} channels but its mapping had {len(mappings)} channels" - ) - if len({m for m in mappings if m > -1}) != max(mappings) + 1: - raise UnityObservationException( - f"Invalid Compressed Channel Mapping: the mapping {mappings} does not have the correct format." - ) - if max(mappings) >= len(image_arrays): - raise UnityObservationException( - f"Invalid Compressed Channel Mapping: the mapping has index larger than the total " - f"number of channels in observation - mapping index {max(mappings)} is" - f"invalid for input observation with {len(image_arrays)} channels." - ) - - processed_image_arrays: List[np.array] = [[] for _ in range(max(mappings) + 1)] - for mapping_idx, img in zip(mappings, image_arrays): - if mapping_idx > -1: - processed_image_arrays[mapping_idx].append(img) - - for i, img_array in enumerate(processed_image_arrays): - processed_image_arrays[i] = np.mean(img_array, axis=0) - img = np.stack(processed_image_arrays, axis=2) - return img - - -def _process_images_num_channels(image_arrays, expected_channels): - """ - Helper function for processing decompressed images with number of expected channels. - This is for old API without mapping provided. Use the first n channel, n=expected_channels. - """ - if expected_channels == 1: - # Convert to grayscale - img = np.mean(image_arrays[0], axis=2) - img = np.reshape(img, [img.shape[0], img.shape[1], 1]) - else: - img = np.concatenate(image_arrays, axis=2) - # We can drop additional channels since they may need to be added to include - # numbers of observation channels not divisible by 3. - actual_channels = list(img.shape)[2] - if actual_channels > expected_channels: - img = img[..., 0:expected_channels] - return img - - -def _check_observations_match_spec( - obs_index: int, - observation_spec: ObservationSpec, - agent_info_list: Collection[AgentInfoProto], -) -> None: - """ - Check that all the observations match the expected size. - This gives a nicer error than a cryptic numpy error later. - """ - expected_obs_shape = tuple(observation_spec.shape) - for agent_info in agent_info_list: - agent_obs_shape = tuple(agent_info.observations[obs_index].shape) - if expected_obs_shape != agent_obs_shape: - raise UnityObservationException( - f"Observation at index={obs_index} for agent with " - f"id={agent_info.id} didn't match the ObservationSpec. " - f"Expected shape {expected_obs_shape} but got {agent_obs_shape}." - ) - - -@timed -def _observation_to_np_array( - obs: ObservationProto, expected_shape: Optional[Iterable[int]] = None -) -> np.ndarray: - """ - Converts observation proto into numpy array of the appropriate size. - :param obs: observation proto to be converted - :param expected_shape: optional shape information, used for sanity checks. - :return: processed numpy array of observation from environment - """ - if expected_shape is not None: - if list(obs.shape) != list(expected_shape): - raise UnityObservationException( - f"Observation did not have the expected shape - got {obs.shape} but expected {expected_shape}" - ) - expected_channels = obs.shape[2] - if obs.compression_type == COMPRESSION_TYPE_NONE: - img = np.array(obs.float_data.data, dtype=np.float32) - img = np.reshape(img, obs.shape) - return img - else: - img = process_pixels( - obs.compressed_data, expected_channels, list(obs.compressed_channel_mapping) - ) - # Compare decompressed image size to observation shape and make sure they match - if list(obs.shape) != list(img.shape): - raise UnityObservationException( - f"Decompressed observation did not have the expected shape - " - f"decompressed had {img.shape} but expected {obs.shape}" - ) - return img - - -@timed -def _process_maybe_compressed_observation( - obs_index: int, - observation_spec: ObservationSpec, - agent_info_list: Collection[AgentInfoProto], -) -> np.ndarray: - shape = cast(Tuple[int, int, int], observation_spec.shape) - if len(agent_info_list) == 0: - return np.zeros((0, shape[0], shape[1], shape[2]), dtype=np.float32) - - try: - batched_visual = [ - _observation_to_np_array(agent_obs.observations[obs_index], shape) - for agent_obs in agent_info_list - ] - except ValueError: - # Try to get a more useful error message - _check_observations_match_spec(obs_index, observation_spec, agent_info_list) - # If that didn't raise anything, raise the original error - raise - return np.array(batched_visual, dtype=np.float32) - - -def _raise_on_nan_and_inf(data: np.array, source: str) -> np.array: - # Check for NaNs or Infinite values in the observation or reward data. - # If there's a NaN in the observations, the np.mean() result will be NaN - # If there's an Infinite value (either sign) then the result will be Inf - # See https://stackoverflow.com/questions/6736590/fast-check-for-nan-in-numpy for background - # Note that a very large values (larger than sqrt(float_max)) will result in an Inf value here - # Raise a Runtime error in the case that NaNs or Infinite values make it into the data. - if data.size == 0: - return data - - d = np.mean(data) - has_nan = np.isnan(d) - has_inf = not np.isfinite(d) - - if has_nan: - raise RuntimeError(f"The {source} provided had NaN values.") - if has_inf: - raise RuntimeError(f"The {source} provided had Infinite values.") - - -@timed -def _process_rank_one_or_two_observation( - obs_index: int, - observation_spec: ObservationSpec, - agent_info_list: Collection[AgentInfoProto], -) -> np.ndarray: - if len(agent_info_list) == 0: - return np.zeros((0,) + observation_spec.shape, dtype=np.float32) - try: - np_obs = np.array( - [ - agent_obs.observations[obs_index].float_data.data - for agent_obs in agent_info_list - ], - dtype=np.float32, - ).reshape((len(agent_info_list),) + observation_spec.shape) - except ValueError: - # Try to get a more useful error message - _check_observations_match_spec(obs_index, observation_spec, agent_info_list) - # If that didn't raise anything, raise the original error - raise - _raise_on_nan_and_inf(np_obs, "observations") - return np_obs - - -@timed -def steps_from_proto( - agent_info_list: Collection[AgentInfoProto], behavior_spec: BehaviorSpec -) -> Tuple[DecisionSteps, TerminalSteps]: - decision_agent_info_list = [ - agent_info for agent_info in agent_info_list if not agent_info.done - ] - terminal_agent_info_list = [ - agent_info for agent_info in agent_info_list if agent_info.done - ] - decision_obs_list: List[np.ndarray] = [] - terminal_obs_list: List[np.ndarray] = [] - for obs_index, observation_spec in enumerate(behavior_spec.observation_specs): - is_visual = len(observation_spec.shape) == 3 - if is_visual: - decision_obs_list.append( - _process_maybe_compressed_observation( - obs_index, observation_spec, decision_agent_info_list - ) - ) - terminal_obs_list.append( - _process_maybe_compressed_observation( - obs_index, observation_spec, terminal_agent_info_list - ) - ) - else: - decision_obs_list.append( - _process_rank_one_or_two_observation( - obs_index, observation_spec, decision_agent_info_list - ) - ) - terminal_obs_list.append( - _process_rank_one_or_two_observation( - obs_index, observation_spec, terminal_agent_info_list - ) - ) - decision_rewards = np.array( - [agent_info.reward for agent_info in decision_agent_info_list], dtype=np.float32 - ) - terminal_rewards = np.array( - [agent_info.reward for agent_info in terminal_agent_info_list], dtype=np.float32 - ) - - decision_group_rewards = np.array( - [agent_info.group_reward for agent_info in decision_agent_info_list], - dtype=np.float32, - ) - terminal_group_rewards = np.array( - [agent_info.group_reward for agent_info in terminal_agent_info_list], - dtype=np.float32, - ) - - _raise_on_nan_and_inf(decision_rewards, "rewards") - _raise_on_nan_and_inf(terminal_rewards, "rewards") - _raise_on_nan_and_inf(decision_group_rewards, "group_rewards") - _raise_on_nan_and_inf(terminal_group_rewards, "group_rewards") - - decision_group_id = [agent_info.group_id for agent_info in decision_agent_info_list] - terminal_group_id = [agent_info.group_id for agent_info in terminal_agent_info_list] - - max_step = np.array( - [agent_info.max_step_reached for agent_info in terminal_agent_info_list], - dtype=np.bool, - ) - decision_agent_id = np.array( - [agent_info.id for agent_info in decision_agent_info_list], dtype=np.int32 - ) - terminal_agent_id = np.array( - [agent_info.id for agent_info in terminal_agent_info_list], dtype=np.int32 - ) - action_mask = None - if behavior_spec.action_spec.discrete_size > 0: - if any( - [agent_info.action_mask is not None] - for agent_info in decision_agent_info_list - ): - n_agents = len(decision_agent_info_list) - a_size = np.sum(behavior_spec.action_spec.discrete_branches) - mask_matrix = np.ones((n_agents, a_size), dtype=np.bool) - for agent_index, agent_info in enumerate(decision_agent_info_list): - if agent_info.action_mask is not None: - if len(agent_info.action_mask) == a_size: - mask_matrix[agent_index, :] = [ - False if agent_info.action_mask[k] else True - for k in range(a_size) - ] - action_mask = (1 - mask_matrix).astype(np.bool) - indices = _generate_split_indices( - behavior_spec.action_spec.discrete_branches - ) - action_mask = np.split(action_mask, indices, axis=1) - return ( - DecisionSteps( - decision_obs_list, - decision_rewards, - decision_agent_id, - action_mask, - decision_group_id, - decision_group_rewards, - ), - TerminalSteps( - terminal_obs_list, - terminal_rewards, - max_step, - terminal_agent_id, - terminal_group_id, - terminal_group_rewards, - ), - ) - - -def _generate_split_indices(dims): - if len(dims) <= 1: - return () - result = (dims[0],) - for i in range(len(dims) - 2): - result += (dims[i + 1] + result[i],) - return result diff --git a/3DOrangeTree/mlagents_envs/side_channel/__init__.py b/3DOrangeTree/mlagents_envs/side_channel/__init__.py deleted file mode 100644 index c9a1f5f..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from mlagents_envs.side_channel.incoming_message import IncomingMessage # noqa -from mlagents_envs.side_channel.outgoing_message import OutgoingMessage # noqa - -from mlagents_envs.side_channel.side_channel import SideChannel # noqa -from mlagents_envs.side_channel.default_training_analytics_side_channel import ( # noqa - DefaultTrainingAnalyticsSideChannel, # noqa -) # noqa diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/__init__.cpython-36.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index 0b925ff..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/__init__.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/__init__.cpython-37.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index 071fb0b..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/default_training_analytics_side_channel.cpython-36.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/default_training_analytics_side_channel.cpython-36.pyc deleted file mode 100644 index 3256c0f..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/default_training_analytics_side_channel.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/default_training_analytics_side_channel.cpython-37.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/default_training_analytics_side_channel.cpython-37.pyc deleted file mode 100644 index fa640a9..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/default_training_analytics_side_channel.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/incoming_message.cpython-36.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/incoming_message.cpython-36.pyc deleted file mode 100644 index 50d26dd..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/incoming_message.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/incoming_message.cpython-37.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/incoming_message.cpython-37.pyc deleted file mode 100644 index e2dca44..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/incoming_message.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/outgoing_message.cpython-36.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/outgoing_message.cpython-36.pyc deleted file mode 100644 index 8a8ef41..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/outgoing_message.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/outgoing_message.cpython-37.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/outgoing_message.cpython-37.pyc deleted file mode 100644 index 8494443..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/outgoing_message.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel.cpython-36.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel.cpython-36.pyc deleted file mode 100644 index eb54468..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel.cpython-37.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel.cpython-37.pyc deleted file mode 100644 index b6c9ec2..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel_manager.cpython-36.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel_manager.cpython-36.pyc deleted file mode 100644 index 5d77a10..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel_manager.cpython-36.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel_manager.cpython-37.pyc b/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel_manager.cpython-37.pyc deleted file mode 100644 index 7aa8134..0000000 Binary files a/3DOrangeTree/mlagents_envs/side_channel/__pycache__/side_channel_manager.cpython-37.pyc and /dev/null differ diff --git a/3DOrangeTree/mlagents_envs/side_channel/default_training_analytics_side_channel.py b/3DOrangeTree/mlagents_envs/side_channel/default_training_analytics_side_channel.py deleted file mode 100644 index a53e686..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/default_training_analytics_side_channel.py +++ /dev/null @@ -1,49 +0,0 @@ -import sys -import uuid -import mlagents_envs - -from mlagents_envs.exception import UnityCommunicationException -from mlagents_envs.side_channel import SideChannel, IncomingMessage, OutgoingMessage -from mlagents_envs.communicator_objects.training_analytics_pb2 import ( - TrainingEnvironmentInitialized, -) -from google.protobuf.any_pb2 import Any - - -class DefaultTrainingAnalyticsSideChannel(SideChannel): - """ - Side channel that sends information about the training to the Unity environment so it can be logged. - """ - - CHANNEL_ID = uuid.UUID("b664a4a9-d86f-5a5f-95cb-e8353a7e8356") - - def __init__(self) -> None: - # >>> uuid.uuid5(uuid.NAMESPACE_URL, "com.unity.ml-agents/TrainingAnalyticsSideChannel") - # UUID('b664a4a9-d86f-5a5f-95cb-e8353a7e8356') - # We purposefully use the SAME side channel as the TrainingAnalyticsSideChannel - - super().__init__(DefaultTrainingAnalyticsSideChannel.CHANNEL_ID) - - def on_message_received(self, msg: IncomingMessage) -> None: - raise UnityCommunicationException( - "The DefaultTrainingAnalyticsSideChannel received a message from Unity, " - + "this should not have happened." - ) - - def environment_initialized(self) -> None: - # Tuple of (major, minor, patch) - vi = sys.version_info - - msg = TrainingEnvironmentInitialized( - python_version=f"{vi[0]}.{vi[1]}.{vi[2]}", - mlagents_version="Custom", - mlagents_envs_version=mlagents_envs.__version__, - torch_version="Unknown", - torch_device_type="Unknown", - ) - any_message = Any() - any_message.Pack(msg) - - env_init_msg = OutgoingMessage() - env_init_msg.set_raw_bytes(any_message.SerializeToString()) # type: ignore - super().queue_message_to_send(env_init_msg) diff --git a/3DOrangeTree/mlagents_envs/side_channel/engine_configuration_channel.py b/3DOrangeTree/mlagents_envs/side_channel/engine_configuration_channel.py deleted file mode 100644 index ce1715b..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/engine_configuration_channel.py +++ /dev/null @@ -1,127 +0,0 @@ -from mlagents_envs.side_channel import SideChannel, OutgoingMessage, IncomingMessage -from mlagents_envs.exception import ( - UnityCommunicationException, - UnitySideChannelException, -) -import uuid -from typing import NamedTuple, Optional -from enum import IntEnum - - -class EngineConfig(NamedTuple): - width: Optional[int] - height: Optional[int] - quality_level: Optional[int] - time_scale: Optional[float] - target_frame_rate: Optional[int] - capture_frame_rate: Optional[int] - - @staticmethod - def default_config(): - return EngineConfig(80, 80, 1, 20.0, -1, 60) - - -class EngineConfigurationChannel(SideChannel): - """ - This is the SideChannel for engine configuration exchange. The data in the - engine configuration is as follows : - - int width; - - int height; - - int qualityLevel; - - float timeScale; - - int targetFrameRate; - - int captureFrameRate; - """ - - class ConfigurationType(IntEnum): - SCREEN_RESOLUTION = 0 - QUALITY_LEVEL = 1 - TIME_SCALE = 2 - TARGET_FRAME_RATE = 3 - CAPTURE_FRAME_RATE = 4 - - def __init__(self) -> None: - super().__init__(uuid.UUID("e951342c-4f7e-11ea-b238-784f4387d1f7")) - - def on_message_received(self, msg: IncomingMessage) -> None: - """ - Is called by the environment to the side channel. Can be called - multiple times per step if multiple messages are meant for that - SideChannel. - Note that Python should never receive an engine configuration from - Unity - """ - raise UnityCommunicationException( - "The EngineConfigurationChannel received a message from Unity, " - + "this should not have happened." - ) - - def set_configuration_parameters( - self, - width: Optional[int] = None, - height: Optional[int] = None, - quality_level: Optional[int] = None, - time_scale: Optional[float] = None, - target_frame_rate: Optional[int] = None, - capture_frame_rate: Optional[int] = None, - ) -> None: - """ - Sets the engine configuration. Takes as input the configurations of the - engine. - :param width: Defines the width of the display. (Must be set alongside height) - :param height: Defines the height of the display. (Must be set alongside width) - :param quality_level: Defines the quality level of the simulation. - :param time_scale: Defines the multiplier for the deltatime in the - simulation. If set to a higher value, time will pass faster in the - simulation but the physics might break. - :param target_frame_rate: Instructs simulation to try to render at a - specified frame rate. - :param capture_frame_rate: Instructs the simulation to consider time between - updates to always be constant, regardless of the actual frame rate. - """ - - if (width is None and height is not None) or ( - width is not None and height is None - ): - raise UnitySideChannelException( - "You cannot set the width/height of the screen resolution without also setting the height/width" - ) - - if width is not None and height is not None: - screen_msg = OutgoingMessage() - screen_msg.write_int32(self.ConfigurationType.SCREEN_RESOLUTION) - screen_msg.write_int32(width) - screen_msg.write_int32(height) - super().queue_message_to_send(screen_msg) - - if quality_level is not None: - quality_level_msg = OutgoingMessage() - quality_level_msg.write_int32(self.ConfigurationType.QUALITY_LEVEL) - quality_level_msg.write_int32(quality_level) - super().queue_message_to_send(quality_level_msg) - - if time_scale is not None: - time_scale_msg = OutgoingMessage() - time_scale_msg.write_int32(self.ConfigurationType.TIME_SCALE) - time_scale_msg.write_float32(time_scale) - super().queue_message_to_send(time_scale_msg) - - if target_frame_rate is not None: - target_frame_rate_msg = OutgoingMessage() - target_frame_rate_msg.write_int32(self.ConfigurationType.TARGET_FRAME_RATE) - target_frame_rate_msg.write_int32(target_frame_rate) - super().queue_message_to_send(target_frame_rate_msg) - - if capture_frame_rate is not None: - capture_frame_rate_msg = OutgoingMessage() - capture_frame_rate_msg.write_int32( - self.ConfigurationType.CAPTURE_FRAME_RATE - ) - capture_frame_rate_msg.write_int32(capture_frame_rate) - super().queue_message_to_send(capture_frame_rate_msg) - - def set_configuration(self, config: EngineConfig) -> None: - """ - Sets the engine configuration. Takes as input an EngineConfig. - """ - self.set_configuration_parameters(**config._asdict()) diff --git a/3DOrangeTree/mlagents_envs/side_channel/environment_parameters_channel.py b/3DOrangeTree/mlagents_envs/side_channel/environment_parameters_channel.py deleted file mode 100644 index ff516a5..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/environment_parameters_channel.py +++ /dev/null @@ -1,100 +0,0 @@ -from mlagents_envs.side_channel import SideChannel, IncomingMessage, OutgoingMessage -from mlagents_envs.exception import UnityCommunicationException -import uuid -from enum import IntEnum -from typing import List, Tuple - - -class EnvironmentParametersChannel(SideChannel): - """ - This is the SideChannel for sending environment parameters to Unity. - You can send parameters to an environment with the command - set_float_parameter. - """ - - class EnvironmentDataTypes(IntEnum): - FLOAT = 0 - SAMPLER = 1 - - class SamplerTypes(IntEnum): - UNIFORM = 0 - GAUSSIAN = 1 - MULTIRANGEUNIFORM = 2 - - def __init__(self) -> None: - channel_id = uuid.UUID("534c891e-810f-11ea-a9d0-822485860400") - super().__init__(channel_id) - - def on_message_received(self, msg: IncomingMessage) -> None: - raise UnityCommunicationException( - "The EnvironmentParametersChannel received a message from Unity, " - + "this should not have happened." - ) - - def set_float_parameter(self, key: str, value: float) -> None: - """ - Sets a float environment parameter in the Unity Environment. - :param key: The string identifier of the parameter. - :param value: The float value of the parameter. - """ - msg = OutgoingMessage() - msg.write_string(key) - msg.write_int32(self.EnvironmentDataTypes.FLOAT) - msg.write_float32(value) - super().queue_message_to_send(msg) - - def set_uniform_sampler_parameters( - self, key: str, min_value: float, max_value: float, seed: int - ) -> None: - """ - Sets a uniform environment parameter sampler. - :param key: The string identifier of the parameter. - :param min_value: The minimum of the sampling distribution. - :param max_value: The maximum of the sampling distribution. - :param seed: The random seed to initialize the sampler. - """ - msg = OutgoingMessage() - msg.write_string(key) - msg.write_int32(self.EnvironmentDataTypes.SAMPLER) - msg.write_int32(seed) - msg.write_int32(self.SamplerTypes.UNIFORM) - msg.write_float32(min_value) - msg.write_float32(max_value) - super().queue_message_to_send(msg) - - def set_gaussian_sampler_parameters( - self, key: str, mean: float, st_dev: float, seed: int - ) -> None: - """ - Sets a gaussian environment parameter sampler. - :param key: The string identifier of the parameter. - :param mean: The mean of the sampling distribution. - :param st_dev: The standard deviation of the sampling distribution. - :param seed: The random seed to initialize the sampler. - """ - msg = OutgoingMessage() - msg.write_string(key) - msg.write_int32(self.EnvironmentDataTypes.SAMPLER) - msg.write_int32(seed) - msg.write_int32(self.SamplerTypes.GAUSSIAN) - msg.write_float32(mean) - msg.write_float32(st_dev) - super().queue_message_to_send(msg) - - def set_multirangeuniform_sampler_parameters( - self, key: str, intervals: List[Tuple[float, float]], seed: int - ) -> None: - """ - Sets a multirangeuniform environment parameter sampler. - :param key: The string identifier of the parameter. - :param intervals: The lists of min and max that define each uniform distribution. - :param seed: The random seed to initialize the sampler. - """ - msg = OutgoingMessage() - msg.write_string(key) - msg.write_int32(self.EnvironmentDataTypes.SAMPLER) - msg.write_int32(seed) - msg.write_int32(self.SamplerTypes.MULTIRANGEUNIFORM) - flattened_intervals = [value for interval in intervals for value in interval] - msg.write_float32_list(flattened_intervals) - super().queue_message_to_send(msg) diff --git a/3DOrangeTree/mlagents_envs/side_channel/float_properties_channel.py b/3DOrangeTree/mlagents_envs/side_channel/float_properties_channel.py deleted file mode 100644 index 6c67011..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/float_properties_channel.py +++ /dev/null @@ -1,62 +0,0 @@ -from mlagents_envs.side_channel import SideChannel, IncomingMessage, OutgoingMessage -import uuid -from typing import Dict, Optional, List - - -class FloatPropertiesChannel(SideChannel): - """ - This is the SideChannel for float properties shared with Unity. - You can modify the float properties of an environment with the commands - set_property, get_property and list_properties. - """ - - def __init__(self, channel_id: uuid.UUID = None) -> None: - self._float_properties: Dict[str, float] = {} - if channel_id is None: - channel_id = uuid.UUID("60ccf7d0-4f7e-11ea-b238-784f4387d1f7") - super().__init__(channel_id) - - def on_message_received(self, msg: IncomingMessage) -> None: - """ - Is called by the environment to the side channel. Can be called - multiple times per step if multiple messages are meant for that - SideChannel. - """ - k = msg.read_string() - v = msg.read_float32() - self._float_properties[k] = v - - def set_property(self, key: str, value: float) -> None: - """ - Sets a property in the Unity Environment. - :param key: The string identifier of the property. - :param value: The float value of the property. - """ - self._float_properties[key] = value - msg = OutgoingMessage() - msg.write_string(key) - msg.write_float32(value) - super().queue_message_to_send(msg) - - def get_property(self, key: str) -> Optional[float]: - """ - Gets a property in the Unity Environment. If the property was not - found, will return None. - :param key: The string identifier of the property. - :return: The float value of the property or None. - """ - return self._float_properties.get(key) - - def list_properties(self) -> List[str]: - """ - Returns a list of all the string identifiers of the properties - currently present in the Unity Environment. - """ - return list(self._float_properties.keys()) - - def get_property_dict_copy(self) -> Dict[str, float]: - """ - Returns a copy of the float properties. - :return: - """ - return dict(self._float_properties) diff --git a/3DOrangeTree/mlagents_envs/side_channel/incoming_message.py b/3DOrangeTree/mlagents_envs/side_channel/incoming_message.py deleted file mode 100644 index 6c00f25..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/incoming_message.py +++ /dev/null @@ -1,93 +0,0 @@ -from typing import List -import struct - - -class IncomingMessage: - """ - Utility class for reading the message written to a SideChannel. - Values must be read in the order they were written. - """ - - def __init__(self, buffer: bytes, offset: int = 0): - """ - Create a new IncomingMessage from the bytes. - """ - self.buffer = buffer - self.offset = offset - - def read_bool(self, default_value: bool = False) -> bool: - """ - Read a boolean value from the message buffer. - :param default_value: Default value to use if the end of the message is reached. - :return: The value read from the message, or the default value if the end was reached. - """ - if self._at_end_of_buffer(): - return default_value - - val = struct.unpack_from(" int: - """ - Read an integer value from the message buffer. - :param default_value: Default value to use if the end of the message is reached. - :return: The value read from the message, or the default value if the end was reached. - """ - if self._at_end_of_buffer(): - return default_value - - val = struct.unpack_from(" float: - """ - Read a float value from the message buffer. - :param default_value: Default value to use if the end of the message is reached. - :return: The value read from the message, or the default value if the end was reached. - """ - if self._at_end_of_buffer(): - return default_value - - val = struct.unpack_from(" List[float]: - """ - Read a list of float values from the message buffer. - :param default_value: Default value to use if the end of the message is reached. - :return: The value read from the message, or the default value if the end was reached. - """ - if self._at_end_of_buffer(): - return [] if default_value is None else default_value - - list_len = self.read_int32() - output = [] - for _ in range(list_len): - output.append(self.read_float32()) - return output - - def read_string(self, default_value: str = "") -> str: - """ - Read a string value from the message buffer. - :param default_value: Default value to use if the end of the message is reached. - :return: The value read from the message, or the default value if the end was reached. - """ - if self._at_end_of_buffer(): - return default_value - - encoded_str_len = self.read_int32() - val = self.buffer[self.offset : self.offset + encoded_str_len].decode("ascii") - self.offset += encoded_str_len - return val - - def get_raw_bytes(self) -> bytes: - """ - Get a copy of the internal bytes used by the message. - """ - return bytearray(self.buffer) - - def _at_end_of_buffer(self) -> bool: - return self.offset >= len(self.buffer) diff --git a/3DOrangeTree/mlagents_envs/side_channel/outgoing_message.py b/3DOrangeTree/mlagents_envs/side_channel/outgoing_message.py deleted file mode 100644 index 83bbe34..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/outgoing_message.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import List -import struct - -from mlagents_envs.logging_util import get_logger - -logger = get_logger(__name__) - - -class OutgoingMessage: - """ - Utility class for forming the message that is written to a SideChannel. - All data is written in little-endian format using the struct module. - """ - - def __init__(self): - """ - Create an OutgoingMessage with an empty buffer. - """ - self.buffer = bytearray() - - def write_bool(self, b: bool) -> None: - """ - Append a boolean value. - """ - self.buffer += struct.pack(" None: - """ - Append an integer value. - """ - self.buffer += struct.pack(" None: - """ - Append a float value. It will be truncated to 32-bit precision. - """ - self.buffer += struct.pack(" None: - """ - Append a list of float values. They will be truncated to 32-bit precision. - """ - self.write_int32(len(float_list)) - for f in float_list: - self.write_float32(f) - - def write_string(self, s: str) -> None: - """ - Append a string value. Internally, it will be encoded to ascii, and the - encoded length will also be written to the message. - """ - encoded_key = s.encode("ascii") - self.write_int32(len(encoded_key)) - self.buffer += encoded_key - - def set_raw_bytes(self, buffer: bytearray) -> None: - """ - Set the internal buffer to a new bytearray. This will overwrite any existing data. - :param buffer: - :return: - """ - if self.buffer: - logger.warning( - "Called set_raw_bytes but the message already has been written to. This will overwrite data." - ) - self.buffer = bytearray(buffer) diff --git a/3DOrangeTree/mlagents_envs/side_channel/raw_bytes_channel.py b/3DOrangeTree/mlagents_envs/side_channel/raw_bytes_channel.py deleted file mode 100644 index d9f748f..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/raw_bytes_channel.py +++ /dev/null @@ -1,39 +0,0 @@ -from mlagents_envs.side_channel import SideChannel, IncomingMessage, OutgoingMessage -from typing import List -import uuid - - -class RawBytesChannel(SideChannel): - """ - This is an example of what the SideChannel for raw bytes exchange would - look like. Is meant to be used for general research purpose. - """ - - def __init__(self, channel_id: uuid.UUID): - self._received_messages: List[bytes] = [] - super().__init__(channel_id) - - def on_message_received(self, msg: IncomingMessage) -> None: - """ - Is called by the environment to the side channel. Can be called - multiple times per step if multiple messages are meant for that - SideChannel. - """ - self._received_messages.append(msg.get_raw_bytes()) - - def get_and_clear_received_messages(self) -> List[bytes]: - """ - returns a list of bytearray received from the environment. - """ - result = list(self._received_messages) - self._received_messages = [] - return result - - def send_raw_data(self, data: bytearray) -> None: - """ - Queues a message to be sent by the environment at the next call to - step. - """ - msg = OutgoingMessage() - msg.set_raw_bytes(data) - super().queue_message_to_send(msg) diff --git a/3DOrangeTree/mlagents_envs/side_channel/side_channel.py b/3DOrangeTree/mlagents_envs/side_channel/side_channel.py deleted file mode 100644 index 469cb51..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/side_channel.py +++ /dev/null @@ -1,46 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List -import uuid - -from mlagents_envs.side_channel import IncomingMessage, OutgoingMessage -from mlagents_envs.logging_util import get_logger - -logger = get_logger(__name__) - - -class SideChannel(ABC): - """ - The side channel just get access to a bytes buffer that will be shared - between C# and Python. For example, We will create a specific side channel - for properties that will be a list of string (fixed size) to float number, - that can be modified by both C# and Python. All side channels are passed - to the Env object at construction. - """ - - def __init__(self, channel_id: uuid.UUID): - self._channel_id: uuid.UUID = channel_id - self.message_queue: List[bytearray] = [] - - def queue_message_to_send(self, msg: OutgoingMessage) -> None: - """ - Queues a message to be sent by the environment at the next call to - step. - """ - self.message_queue.append(msg.buffer) - - @abstractmethod - def on_message_received(self, msg: IncomingMessage) -> None: - """ - Is called by the environment to the side channel. Can be called - multiple times per step if multiple messages are meant for that - SideChannel. - """ - pass - - @property - def channel_id(self) -> uuid.UUID: - """ - :return:The type of side channel used. Will influence how the data is - processed in the environment. - """ - return self._channel_id diff --git a/3DOrangeTree/mlagents_envs/side_channel/side_channel_manager.py b/3DOrangeTree/mlagents_envs/side_channel/side_channel_manager.py deleted file mode 100644 index a7783c6..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/side_channel_manager.py +++ /dev/null @@ -1,81 +0,0 @@ -import uuid -import struct -from typing import Dict, Optional, List -from mlagents_envs.side_channel import SideChannel, IncomingMessage -from mlagents_envs.exception import UnityEnvironmentException -from mlagents_envs.logging_util import get_logger - - -class SideChannelManager: - def __init__(self, side_channels=Optional[List[SideChannel]]): - self._side_channels_dict = self._get_side_channels_dict(side_channels) - - def process_side_channel_message(self, data: bytes) -> None: - """ - Separates the data received from Python into individual messages for each - registered side channel and calls on_message_received on them. - :param data: The packed message sent by Unity - """ - offset = 0 - while offset < len(data): - try: - channel_id = uuid.UUID(bytes_le=bytes(data[offset : offset + 16])) - offset += 16 - message_len, = struct.unpack_from(" bytearray: - """ - Gathers the messages that the registered side channels will send to Unity - and combines them into a single message ready to be sent. - """ - result = bytearray() - for channel_id, channel in self._side_channels_dict.items(): - for message in channel.message_queue: - result += channel_id.bytes_le - result += struct.pack(" Dict[uuid.UUID, SideChannel]: - """ - Converts a list of side channels into a dictionary of channel_id to SideChannel - :param side_channels: The list of side channels. - """ - side_channels_dict: Dict[uuid.UUID, SideChannel] = {} - if side_channels is not None: - for _sc in side_channels: - if _sc.channel_id in side_channels_dict: - raise UnityEnvironmentException( - f"There cannot be two side channels with " - f"the same channel id {_sc.channel_id}." - ) - side_channels_dict[_sc.channel_id] = _sc - return side_channels_dict diff --git a/3DOrangeTree/mlagents_envs/side_channel/stats_side_channel.py b/3DOrangeTree/mlagents_envs/side_channel/stats_side_channel.py deleted file mode 100644 index 9fbbfb2..0000000 --- a/3DOrangeTree/mlagents_envs/side_channel/stats_side_channel.py +++ /dev/null @@ -1,62 +0,0 @@ -import uuid -from typing import Tuple, List, Mapping -from enum import Enum -from collections import defaultdict - -from mlagents_envs.side_channel import SideChannel, IncomingMessage - - -# Determines the behavior of how multiple stats within the same summary period are combined. -class StatsAggregationMethod(Enum): - # Values within the summary period are averaged before reporting. - AVERAGE = 0 - - # Only the most recent value is reported. - MOST_RECENT = 1 - - # Values within the summary period are summed up before reporting. - SUM = 2 - - # All values within a summary period are reported as a histogram. - HISTOGRAM = 3 - - -StatList = List[Tuple[float, StatsAggregationMethod]] -EnvironmentStats = Mapping[str, StatList] - - -class StatsSideChannel(SideChannel): - """ - Side channel that receives (string, float) pairs from the environment, so that they can eventually - be passed to a StatsReporter. - """ - - def __init__(self) -> None: - # >>> uuid.uuid5(uuid.NAMESPACE_URL, "com.unity.ml-agents/StatsSideChannel") - # UUID('a1d8f7b7-cec8-50f9-b78b-d3e165a78520') - super().__init__(uuid.UUID("a1d8f7b7-cec8-50f9-b78b-d3e165a78520")) - - self.stats: EnvironmentStats = defaultdict(list) - - def on_message_received(self, msg: IncomingMessage) -> None: - """ - Receive the message from the environment, and save it for later retrieval. - - :param msg: - :return: - """ - key = msg.read_string() - val = msg.read_float32() - agg_type = StatsAggregationMethod(msg.read_int32()) - - self.stats[key].append((val, agg_type)) - - def get_and_reset_stats(self) -> EnvironmentStats: - """ - Returns the current stats, and resets the internal storage of the stats. - - :return: - """ - s = self.stats - self.stats = defaultdict(list) - return s diff --git a/3DOrangeTree/mlagents_envs/tests/__init__.py b/3DOrangeTree/mlagents_envs/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/3DOrangeTree/mlagents_envs/tests/test_env_utils.py b/3DOrangeTree/mlagents_envs/tests/test_env_utils.py deleted file mode 100644 index e8b783f..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_env_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -from unittest import mock -import pytest -from mlagents_envs.env_utils import validate_environment_path, launch_executable -from mlagents_envs.exception import UnityEnvironmentException -from mlagents_envs.logging_util import ( - set_log_level, - get_logger, - INFO, - ERROR, - FATAL, - CRITICAL, - DEBUG, -) - - -def mock_glob_method(path): - """ - Given a path input, returns a list of candidates - """ - if ".x86" in path: - return ["linux"] - if ".app" in path: - return ["darwin"] - if ".exe" in path: - return ["win32"] - if "*" in path: - return "Any" - return [] - - -@mock.patch("sys.platform") -@mock.patch("glob.glob") -def test_validate_path_empty(glob_mock, platform_mock): - glob_mock.return_value = None - path = validate_environment_path(" ") - assert path is None - - -@mock.patch("mlagents_envs.env_utils.get_platform") -@mock.patch("glob.glob") -def test_validate_path(glob_mock, platform_mock): - glob_mock.side_effect = mock_glob_method - for platform in ["linux", "darwin", "win32"]: - platform_mock.return_value = platform - path = validate_environment_path(" ") - assert path == platform - - -@mock.patch("glob.glob") -@mock.patch("subprocess.Popen") -def test_launch_executable(mock_popen, glob_mock): - with pytest.raises(UnityEnvironmentException): - launch_executable(" ", []) - glob_mock.return_value = ["FakeLaunchPath"] - launch_executable(" ", []) - mock_popen.side_effect = PermissionError("Fake permission error") - with pytest.raises(UnityEnvironmentException): - launch_executable(" ", []) - - -def test_set_logging_level(): - for level in [INFO, ERROR, FATAL, CRITICAL, DEBUG]: - set_log_level(level) - assert get_logger("test").level == level diff --git a/3DOrangeTree/mlagents_envs/tests/test_envs.py b/3DOrangeTree/mlagents_envs/tests/test_envs.py deleted file mode 100644 index 138ac6d..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_envs.py +++ /dev/null @@ -1,170 +0,0 @@ -from unittest import mock -import pytest - -from mlagents_envs.environment import UnityEnvironment -from mlagents_envs.base_env import DecisionSteps, TerminalSteps, ActionTuple -from mlagents_envs.exception import UnityEnvironmentException, UnityActionException -from mlagents_envs.mock_communicator import MockCommunicator - - -@mock.patch("mlagents_envs.environment.UnityEnvironment._get_communicator") -def test_handles_bad_filename(get_communicator): - with pytest.raises(UnityEnvironmentException): - UnityEnvironment(" ") - - -@mock.patch("mlagents_envs.env_utils.launch_executable") -@mock.patch("mlagents_envs.environment.UnityEnvironment._get_communicator") -def test_initialization(mock_communicator, mock_launcher): - mock_communicator.return_value = MockCommunicator( - discrete_action=False, visual_inputs=0 - ) - env = UnityEnvironment(" ") - assert list(env.behavior_specs.keys()) == ["RealFakeBrain"] - env.close() - - -@pytest.mark.parametrize( - "base_port,file_name,expected", - [ - # Non-None base port value will always be used - (6001, "foo.exe", 6001), - # No port specified and environment specified, so use BASE_ENVIRONMENT_PORT - (None, "foo.exe", UnityEnvironment.BASE_ENVIRONMENT_PORT), - # No port specified and no environment, so use DEFAULT_EDITOR_PORT - (None, None, UnityEnvironment.DEFAULT_EDITOR_PORT), - ], -) -@mock.patch("mlagents_envs.env_utils.launch_executable") -@mock.patch("mlagents_envs.environment.UnityEnvironment._get_communicator") -def test_port_defaults( - mock_communicator, mock_launcher, base_port, file_name, expected -): - mock_communicator.return_value = MockCommunicator( - discrete_action=False, visual_inputs=0 - ) - env = UnityEnvironment(file_name=file_name, worker_id=0, base_port=base_port) - assert expected == env._port - env.close() - - -@mock.patch("mlagents_envs.env_utils.launch_executable") -@mock.patch("mlagents_envs.environment.UnityEnvironment._get_communicator") -def test_log_file_path_is_set(mock_communicator, mock_launcher): - mock_communicator.return_value = MockCommunicator() - env = UnityEnvironment( - file_name="myfile", worker_id=0, log_folder="./some-log-folder-path" - ) - args = env._executable_args() - log_file_index = args.index("-logFile") - assert args[log_file_index + 1] == "./some-log-folder-path/Player-0.log" - env.close() - - -@mock.patch("mlagents_envs.env_utils.launch_executable") -@mock.patch("mlagents_envs.environment.UnityEnvironment._get_communicator") -def test_reset(mock_communicator, mock_launcher): - mock_communicator.return_value = MockCommunicator( - discrete_action=False, visual_inputs=0 - ) - env = UnityEnvironment(" ") - spec = env.behavior_specs["RealFakeBrain"] - env.reset() - decision_steps, terminal_steps = env.get_steps("RealFakeBrain") - env.close() - assert isinstance(decision_steps, DecisionSteps) - assert isinstance(terminal_steps, TerminalSteps) - assert len(spec.observation_specs) == len(decision_steps.obs) - assert len(spec.observation_specs) == len(terminal_steps.obs) - n_agents = len(decision_steps) - for sen_spec, obs in zip(spec.observation_specs, decision_steps.obs): - assert (n_agents,) + sen_spec.shape == obs.shape - n_agents = len(terminal_steps) - for sen_spec, obs in zip(spec.observation_specs, terminal_steps.obs): - assert (n_agents,) + sen_spec.shape == obs.shape - - -@mock.patch("mlagents_envs.env_utils.launch_executable") -@mock.patch("mlagents_envs.environment.UnityEnvironment._get_communicator") -def test_step(mock_communicator, mock_launcher): - mock_communicator.return_value = MockCommunicator( - discrete_action=False, visual_inputs=0 - ) - env = UnityEnvironment(" ") - spec = env.behavior_specs["RealFakeBrain"] - env.step() - decision_steps, terminal_steps = env.get_steps("RealFakeBrain") - n_agents = len(decision_steps) - env.set_actions("RealFakeBrain", spec.action_spec.empty_action(n_agents)) - env.step() - with pytest.raises(UnityActionException): - env.set_actions("RealFakeBrain", spec.action_spec.empty_action(n_agents - 1)) - decision_steps, terminal_steps = env.get_steps("RealFakeBrain") - n_agents = len(decision_steps) - _empty_act = spec.action_spec.empty_action(n_agents) - next_action = ActionTuple(_empty_act.continuous - 1, _empty_act.discrete - 1) - env.set_actions("RealFakeBrain", next_action) - env.step() - - env.close() - assert isinstance(decision_steps, DecisionSteps) - assert isinstance(terminal_steps, TerminalSteps) - assert len(spec.observation_specs) == len(decision_steps.obs) - assert len(spec.observation_specs) == len(terminal_steps.obs) - for spec, obs in zip(spec.observation_specs, decision_steps.obs): - assert (n_agents,) + spec.shape == obs.shape - assert 0 in decision_steps - assert 2 in terminal_steps - - -@mock.patch("mlagents_envs.env_utils.launch_executable") -@mock.patch("mlagents_envs.environment.UnityEnvironment._get_communicator") -def test_close(mock_communicator, mock_launcher): - comm = MockCommunicator(discrete_action=False, visual_inputs=0) - mock_communicator.return_value = comm - env = UnityEnvironment(" ") - assert env._loaded - env.close() - assert not env._loaded - assert comm.has_been_closed - - -def test_check_communication_compatibility(): - unity_ver = "1.0.0" - python_ver = "1.0.0" - unity_package_version = "0.15.0" - assert UnityEnvironment._check_communication_compatibility( - unity_ver, python_ver, unity_package_version - ) - unity_ver = "1.1.0" - assert UnityEnvironment._check_communication_compatibility( - unity_ver, python_ver, unity_package_version - ) - unity_ver = "2.0.0" - assert not UnityEnvironment._check_communication_compatibility( - unity_ver, python_ver, unity_package_version - ) - - unity_ver = "0.16.0" - python_ver = "0.16.0" - assert UnityEnvironment._check_communication_compatibility( - unity_ver, python_ver, unity_package_version - ) - unity_ver = "0.17.0" - assert not UnityEnvironment._check_communication_compatibility( - unity_ver, python_ver, unity_package_version - ) - unity_ver = "1.16.0" - assert not UnityEnvironment._check_communication_compatibility( - unity_ver, python_ver, unity_package_version - ) - - -def test_returncode_to_signal_name(): - assert UnityEnvironment._returncode_to_signal_name(-2) == "SIGINT" - assert UnityEnvironment._returncode_to_signal_name(42) is None - assert UnityEnvironment._returncode_to_signal_name("SIGINT") is None - - -if __name__ == "__main__": - pytest.main() diff --git a/3DOrangeTree/mlagents_envs/tests/test_registry.py b/3DOrangeTree/mlagents_envs/tests/test_registry.py deleted file mode 100644 index 12cfefa..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_registry.py +++ /dev/null @@ -1,44 +0,0 @@ -import shutil -import os - -from mlagents_envs.registry import default_registry, UnityEnvRegistry -from mlagents_envs.registry.remote_registry_entry import RemoteRegistryEntry -from mlagents_envs.registry.binary_utils import get_tmp_dir - -BASIC_ID = "Basic" - - -def delete_binaries(): - tmp_dir, bin_dir = get_tmp_dir() - shutil.rmtree(tmp_dir) - shutil.rmtree(bin_dir) - - -def create_registry(): - reg = UnityEnvRegistry() - entry = RemoteRegistryEntry( - BASIC_ID, - 0.0, - "Description", - "https://storage.googleapis.com/mlagents-test-environments/1.0.0/linux/Basic.zip", - "https://storage.googleapis.com/mlagents-test-environments/1.0.0/darwin/Basic.zip", - "https://storage.googleapis.com/mlagents-test-environments/1.0.0/windows/Basic.zip", - ) - reg.register(entry) - return reg - - -def test_basic_in_registry(): - assert BASIC_ID in default_registry - os.environ["TERM"] = "xterm" - delete_binaries() - registry = create_registry() - for worker_id in range(2): - assert BASIC_ID in registry - env = registry[BASIC_ID].make( - base_port=6002, worker_id=worker_id, no_graphics=True - ) - env.reset() - env.step() - assert len(env.behavior_specs) == 1 - env.close() diff --git a/3DOrangeTree/mlagents_envs/tests/test_rpc_communicator.py b/3DOrangeTree/mlagents_envs/tests/test_rpc_communicator.py deleted file mode 100644 index e912878..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_rpc_communicator.py +++ /dev/null @@ -1,82 +0,0 @@ -import pytest -from unittest import mock - -import grpc - -import mlagents_envs.rpc_communicator -from mlagents_envs.rpc_communicator import RpcCommunicator -from mlagents_envs.exception import ( - UnityWorkerInUseException, - UnityTimeOutException, - UnityEnvironmentException, -) -from mlagents_envs.communicator_objects.unity_input_pb2 import UnityInputProto - - -def test_rpc_communicator_checks_port_on_create(): - first_comm = RpcCommunicator() - with pytest.raises(UnityWorkerInUseException): - second_comm = RpcCommunicator() - second_comm.close() - first_comm.close() - - -def test_rpc_communicator_close(): - # Ensures it is possible to open a new RPC Communicators - # after closing one on the same worker_id - first_comm = RpcCommunicator() - first_comm.close() - second_comm = RpcCommunicator() - second_comm.close() - - -def test_rpc_communicator_create_multiple_workers(): - # Ensures multiple RPC communicators can be created with - # different worker_ids without causing an error. - first_comm = RpcCommunicator() - second_comm = RpcCommunicator(worker_id=1) - first_comm.close() - second_comm.close() - - -@mock.patch.object(grpc, "server") -@mock.patch.object( - mlagents_envs.rpc_communicator, "UnityToExternalServicerImplementation" -) -def test_rpc_communicator_initialize_OK(mock_impl, mock_grpc_server): - comm = RpcCommunicator(timeout_wait=0.25) - comm.unity_to_external.parent_conn.poll.return_value = True - input = UnityInputProto() - comm.initialize(input) - comm.unity_to_external.parent_conn.poll.assert_called() - - -@mock.patch.object(grpc, "server") -@mock.patch.object( - mlagents_envs.rpc_communicator, "UnityToExternalServicerImplementation" -) -def test_rpc_communicator_initialize_timeout(mock_impl, mock_grpc_server): - comm = RpcCommunicator(timeout_wait=0.25) - comm.unity_to_external.parent_conn.poll.return_value = None - input = UnityInputProto() - # Expect a timeout - with pytest.raises(UnityTimeOutException): - comm.initialize(input) - comm.unity_to_external.parent_conn.poll.assert_called() - - -@mock.patch.object(grpc, "server") -@mock.patch.object( - mlagents_envs.rpc_communicator, "UnityToExternalServicerImplementation" -) -def test_rpc_communicator_initialize_callback(mock_impl, mock_grpc_server): - def callback(): - raise UnityEnvironmentException - - comm = RpcCommunicator(timeout_wait=0.25) - comm.unity_to_external.parent_conn.poll.return_value = None - input = UnityInputProto() - # Expect a timeout - with pytest.raises(UnityEnvironmentException): - comm.initialize(input, poll_callback=callback) - comm.unity_to_external.parent_conn.poll.assert_called() diff --git a/3DOrangeTree/mlagents_envs/tests/test_rpc_utils.py b/3DOrangeTree/mlagents_envs/tests/test_rpc_utils.py deleted file mode 100644 index 1d03b21..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_rpc_utils.py +++ /dev/null @@ -1,503 +0,0 @@ -import io -import numpy as np -import pytest -from typing import List, Tuple, Any - -from mlagents_envs.communicator_objects.agent_info_pb2 import AgentInfoProto -from mlagents_envs.communicator_objects.observation_pb2 import ( - ObservationProto, - NONE, - PNG, -) -from mlagents_envs.communicator_objects.brain_parameters_pb2 import BrainParametersProto -from mlagents_envs.communicator_objects.agent_info_action_pair_pb2 import ( - AgentInfoActionPairProto, -) -from mlagents_envs.communicator_objects.agent_action_pb2 import AgentActionProto -from mlagents_envs.base_env import ( - BehaviorSpec, - ActionSpec, - DecisionSteps, - TerminalSteps, -) -from mlagents_envs.exception import UnityObservationException -from mlagents_envs.rpc_utils import ( - behavior_spec_from_proto, - process_pixels, - _process_maybe_compressed_observation, - _process_rank_one_or_two_observation, - steps_from_proto, -) -from PIL import Image -from mlagents.trainers.tests.dummy_config import create_observation_specs_with_shapes - - -def generate_list_agent_proto( - n_agent: int, - shape: List[Tuple[int]], - infinite_rewards: bool = False, - nan_observations: bool = False, -) -> List[AgentInfoProto]: - result = [] - for agent_index in range(n_agent): - ap = AgentInfoProto() - ap.reward = float("inf") if infinite_rewards else agent_index - ap.done = agent_index % 2 == 0 - ap.max_step_reached = agent_index % 4 == 0 - ap.id = agent_index - ap.action_mask.extend([True, False] * 5) - obs_proto_list = [] - for obs_index in range(len(shape)): - obs_proto = ObservationProto() - obs_proto.shape.extend(list(shape[obs_index])) - obs_proto.compression_type = NONE - obs_proto.float_data.data.extend( - ([float("nan")] if nan_observations else [0.1]) - * np.prod(shape[obs_index]) - ) - obs_proto_list.append(obs_proto) - ap.observations.extend(obs_proto_list) - result.append(ap) - return result - - -def generate_compressed_data(in_array: np.ndarray) -> bytes: - image_arr = (in_array * 255).astype(np.uint8) - bytes_out = bytes() - - num_channels = in_array.shape[2] - num_images = (num_channels + 2) // 3 - # Split the input image into batches of 3 channels. - for i in range(num_images): - sub_image = image_arr[..., 3 * i : 3 * i + 3] - if (i == num_images - 1) and (num_channels % 3) != 0: - # Pad zeros - zero_shape = list(in_array.shape) - zero_shape[2] = 3 - (num_channels % 3) - z = np.zeros(zero_shape, dtype=np.uint8) - sub_image = np.concatenate([sub_image, z], axis=2) - im = Image.fromarray(sub_image, "RGB") - byteIO = io.BytesIO() - im.save(byteIO, format="PNG") - bytes_out += byteIO.getvalue() - return bytes_out - - -# test helper function for old C# API (no compressed channel mapping) -def generate_compressed_proto_obs( - in_array: np.ndarray, grayscale: bool = False -) -> ObservationProto: - obs_proto = ObservationProto() - obs_proto.compressed_data = generate_compressed_data(in_array) - obs_proto.compression_type = PNG - if grayscale: - # grayscale flag is only used for old API without mapping - expected_shape = [in_array.shape[0], in_array.shape[1], 1] - obs_proto.shape.extend(expected_shape) - else: - obs_proto.shape.extend(in_array.shape) - return obs_proto - - -# test helper function for new C# API (with compressed channel mapping) -def generate_compressed_proto_obs_with_mapping( - in_array: np.ndarray, mapping: List[int] -) -> ObservationProto: - obs_proto = ObservationProto() - obs_proto.compressed_data = generate_compressed_data(in_array) - obs_proto.compression_type = PNG - if mapping is not None: - obs_proto.compressed_channel_mapping.extend(mapping) - expected_shape = [ - in_array.shape[0], - in_array.shape[1], - len({m for m in mapping if m >= 0}), - ] - obs_proto.shape.extend(expected_shape) - else: - obs_proto.shape.extend(in_array.shape) - return obs_proto - - -def generate_uncompressed_proto_obs(in_array: np.ndarray) -> ObservationProto: - obs_proto = ObservationProto() - obs_proto.float_data.data.extend(in_array.flatten().tolist()) - obs_proto.compression_type = NONE - obs_proto.shape.extend(in_array.shape) - return obs_proto - - -def proto_from_steps( - decision_steps: DecisionSteps, terminal_steps: TerminalSteps -) -> List[AgentInfoProto]: - agent_info_protos: List[AgentInfoProto] = [] - # Take care of the DecisionSteps first - for agent_id in decision_steps.agent_id: - agent_id_index = decision_steps.agent_id_to_index[agent_id] - reward = decision_steps.reward[agent_id_index] - done = False - max_step_reached = False - agent_mask: Any = None - if decision_steps.action_mask is not None: - agent_mask = [] - for _branch in decision_steps.action_mask: - agent_mask = np.concatenate( - (agent_mask, _branch[agent_id_index, :]), axis=0 - ) - agent_mask = agent_mask.astype(np.bool).tolist() - observations: List[ObservationProto] = [] - for all_observations_of_type in decision_steps.obs: - observation = all_observations_of_type[agent_id_index] - if len(observation.shape) == 3: - observations.append(generate_uncompressed_proto_obs(observation)) - else: - observations.append( - ObservationProto( - float_data=ObservationProto.FloatData(data=observation), - shape=[len(observation)], - compression_type=NONE, - ) - ) - agent_info_proto = AgentInfoProto( - reward=reward, - done=done, - id=agent_id, - max_step_reached=bool(max_step_reached), - action_mask=agent_mask, - observations=observations, - ) - agent_info_protos.append(agent_info_proto) - # Take care of the TerminalSteps second - for agent_id in terminal_steps.agent_id: - agent_id_index = terminal_steps.agent_id_to_index[agent_id] - reward = terminal_steps.reward[agent_id_index] - done = True - max_step_reached = terminal_steps.interrupted[agent_id_index] - - final_observations: List[ObservationProto] = [] - for all_observations_of_type in terminal_steps.obs: - observation = all_observations_of_type[agent_id_index] - if len(observation.shape) == 3: - final_observations.append(generate_uncompressed_proto_obs(observation)) - else: - final_observations.append( - ObservationProto( - float_data=ObservationProto.FloatData(data=observation), - shape=[len(observation)], - compression_type=NONE, - ) - ) - agent_info_proto = AgentInfoProto( - reward=reward, - done=done, - id=agent_id, - max_step_reached=bool(max_step_reached), - action_mask=None, - observations=final_observations, - ) - agent_info_protos.append(agent_info_proto) - - return agent_info_protos - - -# The arguments here are the DecisionSteps, TerminalSteps and continuous/discrete actions for a single agent name -def proto_from_steps_and_action( - decision_steps: DecisionSteps, - terminal_steps: TerminalSteps, - continuous_actions: np.ndarray, - discrete_actions: np.ndarray, -) -> List[AgentInfoActionPairProto]: - agent_info_protos = proto_from_steps(decision_steps, terminal_steps) - agent_action_protos = [] - num_agents = ( - len(continuous_actions) - if continuous_actions is not None - else len(discrete_actions) - ) - for i in range(num_agents): - proto = AgentActionProto() - if continuous_actions is not None: - proto.continuous_actions.extend(continuous_actions[i]) - proto.vector_actions_deprecated.extend(continuous_actions[i]) - if discrete_actions is not None: - proto.discrete_actions.extend(discrete_actions[i]) - proto.vector_actions_deprecated.extend(discrete_actions[i]) - agent_action_protos.append(proto) - agent_info_action_pair_protos = [ - AgentInfoActionPairProto(agent_info=agent_info_proto, action_info=action_proto) - for agent_info_proto, action_proto in zip( - agent_info_protos, agent_action_protos - ) - ] - return agent_info_action_pair_protos - - -def test_process_pixels(): - in_array = np.random.rand(128, 64, 3) - byte_arr = generate_compressed_data(in_array) - out_array = process_pixels(byte_arr, 3) - assert out_array.shape == (128, 64, 3) - assert np.sum(in_array - out_array) / np.prod(in_array.shape) < 0.01 - assert np.allclose(in_array, out_array, atol=0.01) - - -def test_process_pixels_multi_png(): - height = 128 - width = 64 - num_channels = 7 - in_array = np.random.rand(height, width, num_channels) - byte_arr = generate_compressed_data(in_array) - out_array = process_pixels(byte_arr, num_channels) - assert out_array.shape == (height, width, num_channels) - assert np.sum(in_array - out_array) / np.prod(in_array.shape) < 0.01 - assert np.allclose(in_array, out_array, atol=0.01) - - -def test_process_pixels_gray(): - in_array = np.random.rand(128, 64, 3) - byte_arr = generate_compressed_data(in_array) - out_array = process_pixels(byte_arr, 1) - assert out_array.shape == (128, 64, 1) - assert np.mean(in_array.mean(axis=2, keepdims=True) - out_array) < 0.01 - assert np.allclose(in_array.mean(axis=2, keepdims=True), out_array, atol=0.01) - - -def test_vector_observation(): - n_agents = 10 - shapes = [(3,), (4,)] - obs_specs = create_observation_specs_with_shapes(shapes) - list_proto = generate_list_agent_proto(n_agents, shapes) - for obs_index, shape in enumerate(shapes): - arr = _process_rank_one_or_two_observation( - obs_index, obs_specs[obs_index], list_proto - ) - assert list(arr.shape) == ([n_agents] + list(shape)) - assert np.allclose(arr, 0.1, atol=0.01) - - -def test_process_visual_observation(): - shape = (128, 64, 3) - in_array_1 = np.random.rand(*shape) - proto_obs_1 = generate_compressed_proto_obs(in_array_1) - in_array_2 = np.random.rand(*shape) - in_array_2_mapping = [0, 1, 2] - proto_obs_2 = generate_compressed_proto_obs_with_mapping( - in_array_2, in_array_2_mapping - ) - - ap1 = AgentInfoProto() - ap1.observations.extend([proto_obs_1]) - ap2 = AgentInfoProto() - ap2.observations.extend([proto_obs_2]) - ap_list = [ap1, ap2] - obs_spec = create_observation_specs_with_shapes([shape])[0] - arr = _process_maybe_compressed_observation(0, obs_spec, ap_list) - assert list(arr.shape) == [2, 128, 64, 3] - assert np.allclose(arr[0, :, :, :], in_array_1, atol=0.01) - assert np.allclose(arr[1, :, :, :], in_array_2, atol=0.01) - - -def test_process_visual_observation_grayscale(): - in_array_1 = np.random.rand(128, 64, 3) - proto_obs_1 = generate_compressed_proto_obs(in_array_1, grayscale=True) - expected_out_array_1 = np.mean(in_array_1, axis=2, keepdims=True) - in_array_2 = np.random.rand(128, 64, 3) - in_array_2_mapping = [0, 0, 0] - proto_obs_2 = generate_compressed_proto_obs_with_mapping( - in_array_2, in_array_2_mapping - ) - expected_out_array_2 = np.mean(in_array_2, axis=2, keepdims=True) - - ap1 = AgentInfoProto() - ap1.observations.extend([proto_obs_1]) - ap2 = AgentInfoProto() - ap2.observations.extend([proto_obs_2]) - ap_list = [ap1, ap2] - shape = (128, 64, 1) - obs_spec = create_observation_specs_with_shapes([shape])[0] - arr = _process_maybe_compressed_observation(0, obs_spec, ap_list) - assert list(arr.shape) == [2, 128, 64, 1] - assert np.allclose(arr[0, :, :, :], expected_out_array_1, atol=0.01) - assert np.allclose(arr[1, :, :, :], expected_out_array_2, atol=0.01) - - -def test_process_visual_observation_padded_channels(): - in_array_1 = np.random.rand(128, 64, 12) - in_array_1_mapping = [0, 1, 2, 3, -1, -1, 4, 5, 6, 7, -1, -1] - proto_obs_1 = generate_compressed_proto_obs_with_mapping( - in_array_1, in_array_1_mapping - ) - expected_out_array_1 = np.take(in_array_1, [0, 1, 2, 3, 6, 7, 8, 9], axis=2) - - ap1 = AgentInfoProto() - ap1.observations.extend([proto_obs_1]) - ap_list = [ap1] - shape = (128, 64, 8) - obs_spec = create_observation_specs_with_shapes([shape])[0] - - arr = _process_maybe_compressed_observation(0, obs_spec, ap_list) - assert list(arr.shape) == [1, 128, 64, 8] - assert np.allclose(arr[0, :, :, :], expected_out_array_1, atol=0.01) - - -def test_process_visual_observation_bad_shape(): - in_array_1 = np.random.rand(128, 64, 3) - proto_obs_1 = generate_compressed_proto_obs(in_array_1) - ap1 = AgentInfoProto() - ap1.observations.extend([proto_obs_1]) - ap_list = [ap1] - - shape = (128, 42, 3) - obs_spec = create_observation_specs_with_shapes([shape])[0] - - with pytest.raises(UnityObservationException): - _process_maybe_compressed_observation(0, obs_spec, ap_list) - - -def test_batched_step_result_from_proto(): - n_agents = 10 - shapes = [(3,), (4,)] - spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), ActionSpec.create_continuous(3) - ) - ap_list = generate_list_agent_proto(n_agents, shapes) - decision_steps, terminal_steps = steps_from_proto(ap_list, spec) - for agent_id in range(n_agents): - if agent_id in decision_steps: - # we set the reward equal to the agent id in generate_list_agent_proto - assert decision_steps[agent_id].reward == agent_id - elif agent_id in terminal_steps: - assert terminal_steps[agent_id].reward == agent_id - else: - raise Exception("Missing agent from the steps") - # We sort the AgentId since they are split between DecisionSteps and TerminalSteps - combined_agent_id = list(decision_steps.agent_id) + list(terminal_steps.agent_id) - combined_agent_id.sort() - assert combined_agent_id == list(range(n_agents)) - for agent_id in range(n_agents): - assert (agent_id in terminal_steps) == (agent_id % 2 == 0) - if agent_id in terminal_steps: - assert terminal_steps[agent_id].interrupted == (agent_id % 4 == 0) - assert decision_steps.obs[0].shape[1] == shapes[0][0] - assert decision_steps.obs[1].shape[1] == shapes[1][0] - assert terminal_steps.obs[0].shape[1] == shapes[0][0] - assert terminal_steps.obs[1].shape[1] == shapes[1][0] - - -def test_mismatch_observations_raise_in_step_result_from_proto(): - n_agents = 10 - shapes = [(3,), (4,)] - spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), ActionSpec.create_continuous(3) - ) - ap_list = generate_list_agent_proto(n_agents, shapes) - # Hack an observation to be larger, we should get an exception - ap_list[0].observations[0].shape[0] += 1 - ap_list[0].observations[0].float_data.data.append(0.42) - with pytest.raises(UnityObservationException): - steps_from_proto(ap_list, spec) - - -def test_action_masking_discrete(): - n_agents = 10 - shapes = [(3,), (4,)] - behavior_spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), ActionSpec.create_discrete((7, 3)) - ) - ap_list = generate_list_agent_proto(n_agents, shapes) - decision_steps, terminal_steps = steps_from_proto(ap_list, behavior_spec) - masks = decision_steps.action_mask - assert isinstance(masks, list) - assert len(masks) == 2 - assert masks[0].shape == (n_agents / 2, 7) # half agents are done - assert masks[1].shape == (n_agents / 2, 3) # half agents are done - assert masks[0][0, 0] - assert not masks[1][0, 0] - assert masks[1][0, 1] - - -def test_action_masking_discrete_1(): - n_agents = 10 - shapes = [(3,), (4,)] - behavior_spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), ActionSpec.create_discrete((10,)) - ) - ap_list = generate_list_agent_proto(n_agents, shapes) - decision_steps, terminal_steps = steps_from_proto(ap_list, behavior_spec) - masks = decision_steps.action_mask - assert isinstance(masks, list) - assert len(masks) == 1 - assert masks[0].shape == (n_agents / 2, 10) - assert masks[0][0, 0] - - -def test_action_masking_discrete_2(): - n_agents = 10 - shapes = [(3,), (4,)] - behavior_spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), - ActionSpec.create_discrete((2, 2, 6)), - ) - ap_list = generate_list_agent_proto(n_agents, shapes) - decision_steps, terminal_steps = steps_from_proto(ap_list, behavior_spec) - masks = decision_steps.action_mask - assert isinstance(masks, list) - assert len(masks) == 3 - assert masks[0].shape == (n_agents / 2, 2) - assert masks[1].shape == (n_agents / 2, 2) - assert masks[2].shape == (n_agents / 2, 6) - assert masks[0][0, 0] - - -def test_action_masking_continuous(): - n_agents = 10 - shapes = [(3,), (4,)] - behavior_spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), ActionSpec.create_continuous(10) - ) - ap_list = generate_list_agent_proto(n_agents, shapes) - decision_steps, terminal_steps = steps_from_proto(ap_list, behavior_spec) - masks = decision_steps.action_mask - assert masks is None - - -def test_agent_behavior_spec_from_proto(): - agent_proto = generate_list_agent_proto(1, [(3,), (4,)])[0] - bp = BrainParametersProto() - bp.vector_action_size_deprecated.extend([5, 4]) - bp.vector_action_space_type_deprecated = 0 - behavior_spec = behavior_spec_from_proto(bp, agent_proto) - assert behavior_spec.action_spec.is_discrete() - assert not behavior_spec.action_spec.is_continuous() - assert [spec.shape for spec in behavior_spec.observation_specs] == [(3,), (4,)] - assert behavior_spec.action_spec.discrete_branches == (5, 4) - assert behavior_spec.action_spec.discrete_size == 2 - bp = BrainParametersProto() - bp.vector_action_size_deprecated.extend([6]) - bp.vector_action_space_type_deprecated = 1 - behavior_spec = behavior_spec_from_proto(bp, agent_proto) - assert not behavior_spec.action_spec.is_discrete() - assert behavior_spec.action_spec.is_continuous() - assert behavior_spec.action_spec.continuous_size == 6 - - -def test_batched_step_result_from_proto_raises_on_infinite(): - n_agents = 10 - shapes = [(3,), (4,)] - behavior_spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), ActionSpec.create_continuous(3) - ) - ap_list = generate_list_agent_proto(n_agents, shapes, infinite_rewards=True) - with pytest.raises(RuntimeError): - steps_from_proto(ap_list, behavior_spec) - - -def test_batched_step_result_from_proto_raises_on_nan(): - n_agents = 10 - shapes = [(3,), (4,)] - behavior_spec = BehaviorSpec( - create_observation_specs_with_shapes(shapes), ActionSpec.create_continuous(3) - ) - ap_list = generate_list_agent_proto(n_agents, shapes, nan_observations=True) - with pytest.raises(RuntimeError): - steps_from_proto(ap_list, behavior_spec) diff --git a/3DOrangeTree/mlagents_envs/tests/test_set_action.py b/3DOrangeTree/mlagents_envs/tests/test_set_action.py deleted file mode 100644 index 227e074..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_set_action.py +++ /dev/null @@ -1,55 +0,0 @@ -from mlagents_envs.registry import default_registry -from mlagents_envs.side_channel.engine_configuration_channel import ( - EngineConfigurationChannel, -) -from mlagents_envs.base_env import ActionTuple -import numpy as np - -BALL_ID = "3DBall" - - -def test_set_action_single_agent(): - engine_config_channel = EngineConfigurationChannel() - env = default_registry[BALL_ID].make( - base_port=6000, - worker_id=0, - no_graphics=True, - side_channels=[engine_config_channel], - ) - engine_config_channel.set_configuration_parameters(time_scale=100) - for _ in range(3): - env.reset() - behavior_name = list(env.behavior_specs.keys())[0] - d, t = env.get_steps(behavior_name) - for _ in range(50): - for agent_id in d.agent_id: - action = np.ones((1, 2)) - action_tuple = ActionTuple() - action_tuple.add_continuous(action) - env.set_action_for_agent(behavior_name, agent_id, action_tuple) - env.step() - d, t = env.get_steps(behavior_name) - env.close() - - -def test_set_action_multi_agent(): - engine_config_channel = EngineConfigurationChannel() - env = default_registry[BALL_ID].make( - base_port=6001, - worker_id=0, - no_graphics=True, - side_channels=[engine_config_channel], - ) - engine_config_channel.set_configuration_parameters(time_scale=100) - for _ in range(3): - env.reset() - behavior_name = list(env.behavior_specs.keys())[0] - d, t = env.get_steps(behavior_name) - for _ in range(50): - action = np.ones((len(d), 2)) - action_tuple = ActionTuple() - action_tuple.add_continuous(action) - env.set_actions(behavior_name, action_tuple) - env.step() - d, t = env.get_steps(behavior_name) - env.close() diff --git a/3DOrangeTree/mlagents_envs/tests/test_side_channel.py b/3DOrangeTree/mlagents_envs/tests/test_side_channel.py deleted file mode 100644 index f9b1447..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_side_channel.py +++ /dev/null @@ -1,258 +0,0 @@ -import uuid -import pytest -from mlagents_envs.side_channel import SideChannel, IncomingMessage, OutgoingMessage -from mlagents_envs.side_channel.side_channel_manager import SideChannelManager -from mlagents_envs.side_channel.float_properties_channel import FloatPropertiesChannel -from mlagents_envs.side_channel.raw_bytes_channel import RawBytesChannel -from mlagents_envs.side_channel.engine_configuration_channel import ( - EngineConfigurationChannel, - EngineConfig, -) -from mlagents_envs.side_channel.environment_parameters_channel import ( - EnvironmentParametersChannel, -) -from mlagents_envs.side_channel.stats_side_channel import ( - StatsSideChannel, - StatsAggregationMethod, -) -from mlagents_envs.exception import ( - UnitySideChannelException, - UnityCommunicationException, -) - - -class IntChannel(SideChannel): - def __init__(self): - self.list_int = [] - super().__init__(uuid.UUID("a85ba5c0-4f87-11ea-a517-784f4387d1f7")) - - def on_message_received(self, msg: IncomingMessage) -> None: - val = msg.read_int32() - self.list_int += [val] - - def send_int(self, value): - msg = OutgoingMessage() - msg.write_int32(value) - super().queue_message_to_send(msg) - - -def test_int_channel(): - sender = IntChannel() - receiver = IntChannel() - sender.send_int(5) - sender.send_int(6) - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - assert receiver.list_int[0] == 5 - assert receiver.list_int[1] == 6 - - -def test_float_properties(): - sender = FloatPropertiesChannel() - receiver = FloatPropertiesChannel() - - sender.set_property("prop1", 1.0) - - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - - val = receiver.get_property("prop1") - assert val == 1.0 - val = receiver.get_property("prop2") - assert val is None - sender.set_property("prop2", 2.0) - - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - - val = receiver.get_property("prop1") - assert val == 1.0 - val = receiver.get_property("prop2") - assert val == 2.0 - assert len(receiver.list_properties()) == 2 - assert "prop1" in receiver.list_properties() - assert "prop2" in receiver.list_properties() - val = sender.get_property("prop1") - assert val == 1.0 - - assert receiver.get_property_dict_copy() == {"prop1": 1.0, "prop2": 2.0} - assert receiver.get_property_dict_copy() == sender.get_property_dict_copy() - - -def test_raw_bytes(): - guid = uuid.uuid4() - sender = RawBytesChannel(guid) - receiver = RawBytesChannel(guid) - - sender.send_raw_data(b"foo") - sender.send_raw_data(b"bar") - - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - - messages = receiver.get_and_clear_received_messages() - assert len(messages) == 2 - assert messages[0].decode("ascii") == "foo" - assert messages[1].decode("ascii") == "bar" - - messages = receiver.get_and_clear_received_messages() - assert len(messages) == 0 - - -def test_message_bool(): - vals = [True, False] - msg_out = OutgoingMessage() - for v in vals: - msg_out.write_bool(v) - - msg_in = IncomingMessage(msg_out.buffer) - read_vals = [] - for _ in range(len(vals)): - read_vals.append(msg_in.read_bool()) - assert vals == read_vals - - # Test reading with defaults - assert msg_in.read_bool() is False - assert msg_in.read_bool(default_value=True) is True - - -def test_message_int32(): - val = 1337 - msg_out = OutgoingMessage() - msg_out.write_int32(val) - - msg_in = IncomingMessage(msg_out.buffer) - read_val = msg_in.read_int32() - assert val == read_val - - # Test reading with defaults - assert 0 == msg_in.read_int32() - assert val == msg_in.read_int32(default_value=val) - - -def test_message_float32(): - val = 42.0 - msg_out = OutgoingMessage() - msg_out.write_float32(val) - - msg_in = IncomingMessage(msg_out.buffer) - read_val = msg_in.read_float32() - # These won't be exactly equal in general, since python floats are 64-bit. - assert val == read_val - - # Test reading with defaults - assert 0.0 == msg_in.read_float32() - assert val == msg_in.read_float32(default_value=val) - - -def test_message_string(): - val = "mlagents!" - msg_out = OutgoingMessage() - msg_out.write_string(val) - - msg_in = IncomingMessage(msg_out.buffer) - read_val = msg_in.read_string() - assert val == read_val - - # Test reading with defaults - assert "" == msg_in.read_string() - assert val == msg_in.read_string(default_value=val) - - -def test_message_float_list(): - val = [1.0, 3.0, 9.0] - msg_out = OutgoingMessage() - msg_out.write_float32_list(val) - - msg_in = IncomingMessage(msg_out.buffer) - read_val = msg_in.read_float32_list() - # These won't be exactly equal in general, since python floats are 64-bit. - assert val == read_val - - # Test reading with defaults - assert [] == msg_in.read_float32_list() - assert val == msg_in.read_float32_list(default_value=val) - - -def test_engine_configuration(): - sender = EngineConfigurationChannel() - # We use a raw bytes channel to interpred the data - receiver = RawBytesChannel(sender.channel_id) - - config = EngineConfig.default_config() - sender.set_configuration(config) - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - - received_data = receiver.get_and_clear_received_messages() - assert len(received_data) == 5 # 5 different messages one for each setting - - sent_time_scale = 4.5 - sender.set_configuration_parameters(time_scale=sent_time_scale) - - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - - message = IncomingMessage(receiver.get_and_clear_received_messages()[0]) - message.read_int32() - time_scale = message.read_float32() - assert time_scale == sent_time_scale - - with pytest.raises(UnitySideChannelException): - sender.set_configuration_parameters(width=None, height=42) - - with pytest.raises(UnityCommunicationException): - # try to send data to the EngineConfigurationChannel - sender.set_configuration_parameters(time_scale=sent_time_scale) - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([sender]).process_side_channel_message(data) - - -def test_environment_parameters(): - sender = EnvironmentParametersChannel() - # We use a raw bytes channel to interpred the data - receiver = RawBytesChannel(sender.channel_id) - - sender.set_float_parameter("param-1", 0.1) - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - - message = IncomingMessage(receiver.get_and_clear_received_messages()[0]) - key = message.read_string() - dtype = message.read_int32() - value = message.read_float32() - assert key == "param-1" - assert dtype == EnvironmentParametersChannel.EnvironmentDataTypes.FLOAT - assert value - 0.1 < 1e-8 - - sender.set_float_parameter("param-1", 0.1) - sender.set_float_parameter("param-2", 0.1) - sender.set_float_parameter("param-3", 0.1) - - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([receiver]).process_side_channel_message(data) - - assert len(receiver.get_and_clear_received_messages()) == 3 - - with pytest.raises(UnityCommunicationException): - # try to send data to the EngineConfigurationChannel - sender.set_float_parameter("param-1", 0.1) - data = SideChannelManager([sender]).generate_side_channel_messages() - SideChannelManager([sender]).process_side_channel_message(data) - - -def test_stats_channel(): - receiver = StatsSideChannel() - message = OutgoingMessage() - message.write_string("stats-1") - message.write_float32(42.0) - message.write_int32(1) # corresponds to StatsAggregationMethod.MOST_RECENT - - receiver.on_message_received(IncomingMessage(message.buffer)) - - stats = receiver.get_and_reset_stats() - - assert len(stats) == 1 - val, method = stats["stats-1"][0] - assert val - 42.0 < 1e-8 - assert method == StatsAggregationMethod.MOST_RECENT diff --git a/3DOrangeTree/mlagents_envs/tests/test_steps.py b/3DOrangeTree/mlagents_envs/tests/test_steps.py deleted file mode 100644 index 0160380..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_steps.py +++ /dev/null @@ -1,136 +0,0 @@ -import pytest -import numpy as np - -from mlagents_envs.base_env import ( - DecisionSteps, - TerminalSteps, - ActionSpec, - BehaviorSpec, -) -from mlagents.trainers.tests.dummy_config import create_observation_specs_with_shapes - - -def test_decision_steps(): - ds = DecisionSteps( - obs=[np.array(range(12), dtype=np.float32).reshape(3, 4)], - reward=np.array(range(3), dtype=np.float32), - agent_id=np.array(range(10, 13), dtype=np.int32), - action_mask=[np.zeros((3, 4), dtype=np.bool)], - group_id=np.array(range(3), dtype=np.int32), - group_reward=np.array(range(3), dtype=np.float32), - ) - - assert ds.agent_id_to_index[10] == 0 - assert ds.agent_id_to_index[11] == 1 - assert ds.agent_id_to_index[12] == 2 - - with pytest.raises(KeyError): - assert ds.agent_id_to_index[-1] == -1 - - mask_agent = ds[10].action_mask - assert isinstance(mask_agent, list) - assert len(mask_agent) == 1 - assert np.array_equal(mask_agent[0], np.zeros((4), dtype=np.bool)) - - for agent_id in ds: - assert ds.agent_id_to_index[agent_id] in range(3) - - -def test_empty_decision_steps(): - specs = BehaviorSpec( - observation_specs=create_observation_specs_with_shapes([(3, 2), (5,)]), - action_spec=ActionSpec.create_continuous(3), - ) - ds = DecisionSteps.empty(specs) - assert len(ds.obs) == 2 - assert ds.obs[0].shape == (0, 3, 2) - assert ds.obs[1].shape == (0, 5) - - -def test_terminal_steps(): - ts = TerminalSteps( - obs=[np.array(range(12), dtype=np.float32).reshape(3, 4)], - reward=np.array(range(3), dtype=np.float32), - agent_id=np.array(range(10, 13), dtype=np.int32), - interrupted=np.array([1, 0, 1], dtype=np.bool), - group_id=np.array(range(3), dtype=np.int32), - group_reward=np.array(range(3), dtype=np.float32), - ) - - assert ts.agent_id_to_index[10] == 0 - assert ts.agent_id_to_index[11] == 1 - assert ts.agent_id_to_index[12] == 2 - - assert ts[10].interrupted - assert not ts[11].interrupted - assert ts[12].interrupted - - with pytest.raises(KeyError): - assert ts.agent_id_to_index[-1] == -1 - - for agent_id in ts: - assert ts.agent_id_to_index[agent_id] in range(3) - - -def test_empty_terminal_steps(): - specs = BehaviorSpec( - observation_specs=create_observation_specs_with_shapes([(3, 2), (5,)]), - action_spec=ActionSpec.create_continuous(3), - ) - ts = TerminalSteps.empty(specs) - assert len(ts.obs) == 2 - assert ts.obs[0].shape == (0, 3, 2) - assert ts.obs[1].shape == (0, 5) - - -def test_specs(): - specs = ActionSpec.create_continuous(3) - assert specs.discrete_branches == () - assert specs.discrete_size == 0 - assert specs.continuous_size == 3 - assert specs.empty_action(5).continuous.shape == (5, 3) - assert specs.empty_action(5).continuous.dtype == np.float32 - - specs = ActionSpec.create_discrete((3,)) - assert specs.discrete_branches == (3,) - assert specs.discrete_size == 1 - assert specs.continuous_size == 0 - assert specs.empty_action(5).discrete.shape == (5, 1) - assert specs.empty_action(5).discrete.dtype == np.int32 - - specs = ActionSpec(3, (3,)) - assert specs.continuous_size == 3 - assert specs.discrete_branches == (3,) - assert specs.discrete_size == 1 - assert specs.empty_action(5).continuous.shape == (5, 3) - assert specs.empty_action(5).continuous.dtype == np.float32 - assert specs.empty_action(5).discrete.shape == (5, 1) - assert specs.empty_action(5).discrete.dtype == np.int32 - - -def test_action_generator(): - # Continuous - action_len = 30 - specs = ActionSpec.create_continuous(action_len) - zero_action = specs.empty_action(4).continuous - assert np.array_equal(zero_action, np.zeros((4, action_len), dtype=np.float32)) - print(specs.random_action(4)) - random_action = specs.random_action(4).continuous - print(random_action) - assert random_action.dtype == np.float32 - assert random_action.shape == (4, action_len) - assert np.min(random_action) >= -1 - assert np.max(random_action) <= 1 - - # Discrete - action_shape = (10, 20, 30) - specs = ActionSpec.create_discrete(action_shape) - zero_action = specs.empty_action(4).discrete - assert np.array_equal(zero_action, np.zeros((4, len(action_shape)), dtype=np.int32)) - - random_action = specs.random_action(4).discrete - assert random_action.dtype == np.int32 - assert random_action.shape == (4, len(action_shape)) - assert np.min(random_action) >= 0 - for index, branch_size in enumerate(action_shape): - assert np.max(random_action[:, index]) < branch_size diff --git a/3DOrangeTree/mlagents_envs/tests/test_timers.py b/3DOrangeTree/mlagents_envs/tests/test_timers.py deleted file mode 100644 index 38de561..0000000 --- a/3DOrangeTree/mlagents_envs/tests/test_timers.py +++ /dev/null @@ -1,92 +0,0 @@ -from unittest import mock - -from mlagents_envs import timers - - -@timers.timed -def decorated_func(x: int = 0, y: float = 1.0) -> str: - timers.set_gauge("my_gauge", x + y) - return f"{x} + {y} = {x + y}" - - -def test_timers() -> None: - test_timer = timers.TimerStack() - with mock.patch("mlagents_envs.timers._get_thread_timer", return_value=test_timer): - # First, run some simple code - with timers.hierarchical_timer("top_level"): - for i in range(3): - with timers.hierarchical_timer("multiple"): - decorated_func(i, i) - - raised = False - try: - with timers.hierarchical_timer("raises"): - raise RuntimeError("timeout!") - except RuntimeError: - raised = True - - with timers.hierarchical_timer("post_raise"): - assert raised - pass - - # We expect the hierarchy to look like - # (root) - # top_level - # multiple - # decorated_func - # raises - # post_raise - root = test_timer.root - assert root.children.keys() == {"top_level"} - - top_level = root.children["top_level"] - assert top_level.children.keys() == {"multiple", "raises", "post_raise"} - - # make sure the scope was closed properly when the exception was raised - raises = top_level.children["raises"] - assert raises.count == 1 - - multiple = top_level.children["multiple"] - assert multiple.count == 3 - - timer_tree = test_timer.get_timing_tree() - - expected_tree = { - "name": "root", - "total": mock.ANY, - "count": 1, - "self": mock.ANY, - "children": { - "top_level": { - "total": mock.ANY, - "count": 1, - "self": mock.ANY, - "children": { - "multiple": { - "total": mock.ANY, - "count": 3, - "self": mock.ANY, - "children": { - "decorated_func": { - "total": mock.ANY, - "count": 3, - "self": mock.ANY, - } - }, - }, - "raises": {"total": mock.ANY, "count": 1, "self": mock.ANY}, - "post_raise": {"total": mock.ANY, "count": 1, "self": mock.ANY}, - }, - } - }, - "gauges": {"my_gauge": {"value": 4.0, "max": 4.0, "min": 0.0, "count": 3}}, - "metadata": { - "timer_format_version": timers.TIMER_FORMAT_VERSION, - "start_time_seconds": mock.ANY, - "end_time_seconds": mock.ANY, - "python_version": mock.ANY, - "command_line_arguments": mock.ANY, - }, - } - - assert timer_tree == expected_tree diff --git a/3DOrangeTree/mlagents_envs/timers.py b/3DOrangeTree/mlagents_envs/timers.py deleted file mode 100644 index 32b8602..0000000 --- a/3DOrangeTree/mlagents_envs/timers.py +++ /dev/null @@ -1,362 +0,0 @@ -""" -Lightweight, hierarchical timers for profiling sections of code. - -Example: - -@timed -def foo(t): - time.sleep(t) - -def main(): - for i in range(3): - foo(i + 1) - with hierarchical_timer("context"): - foo(1) - - print(get_timer_tree()) - -This would produce a timer tree like - (root) - "foo" - "context" - "foo" - -The total time and counts are tracked for each block of code; in this example "foo" and "context.foo" are considered -distinct blocks, and are tracked separately. - -The decorator and contextmanager are equivalent; the context manager may be more useful if you want more control -over the timer name, or are splitting up multiple sections of a large function. -""" - -import math -import sys -import time -import threading - -from contextlib import contextmanager -from typing import Any, Callable, Dict, Generator, Optional, TypeVar - -TIMER_FORMAT_VERSION = "0.1.0" - - -class TimerNode: - """ - Represents the time spent in a block of code. - """ - - __slots__ = ["children", "total", "count", "is_parallel"] - - def __init__(self): - # Note that since dictionary keys are the node names, we don't explicitly store the name on the TimerNode. - self.children: Dict[str, TimerNode] = {} - self.total: float = 0.0 - self.count: int = 0 - self.is_parallel = False - - def get_child(self, name: str) -> "TimerNode": - """ - Get the child node corresponding to the name (and create if it doesn't already exist). - """ - child = self.children.get(name) - if child is None: - child = TimerNode() - self.children[name] = child - return child - - def add_time(self, elapsed: float) -> None: - """ - Accumulate the time spent in the node (and increment the count). - """ - self.total += elapsed - self.count += 1 - - def merge( - self, other: "TimerNode", root_name: str = None, is_parallel: bool = True - ) -> None: - """ - Add the other node to this node, then do the same recursively on its children. - :param other: The other node to merge - :param root_name: Optional name of the root node being merged. - :param is_parallel: Whether or not the code block was executed in parallel. - :return: - """ - if root_name: - node = self.get_child(root_name) - else: - node = self - - node.total += other.total - node.count += other.count - node.is_parallel |= is_parallel - for other_child_name, other_child_node in other.children.items(): - child = node.get_child(other_child_name) - child.merge(other_child_node, is_parallel=is_parallel) - - -class GaugeNode: - """ - Tracks the most recent value of a metric. This is analogous to gauges in statsd. - """ - - __slots__ = ["value", "min_value", "max_value", "count", "_timestamp"] - - def __init__(self, value: float): - self.value = value - self.min_value = value - self.max_value = value - self.count = 1 - # Internal timestamp so we can determine priority. - self._timestamp = time.time() - - def update(self, new_value: float) -> None: - self.min_value = min(self.min_value, new_value) - self.max_value = max(self.max_value, new_value) - self.value = new_value - self.count += 1 - self._timestamp = time.time() - - def merge(self, other: "GaugeNode") -> None: - if self._timestamp < other._timestamp: - # Keep the "later" value - self.value = other.value - self._timestamp = other._timestamp - self.min_value = min(self.min_value, other.min_value) - self.max_value = max(self.max_value, other.max_value) - self.count += other.count - - def as_dict(self) -> Dict[str, float]: - return { - "value": self.value, - "min": self.min_value, - "max": self.max_value, - "count": self.count, - } - - -class TimerStack: - """ - Tracks all the time spent. Users shouldn't use this directly, they should use the contextmanager below to make - sure that pushes and pops are already matched. - """ - - __slots__ = ["root", "stack", "start_time", "gauges", "metadata"] - - def __init__(self): - self.root = TimerNode() - self.stack = [self.root] - self.start_time = time.perf_counter() - self.gauges: Dict[str, GaugeNode] = {} - self.metadata: Dict[str, str] = {} - self._add_default_metadata() - - def reset(self): - self.root = TimerNode() - self.stack = [self.root] - self.start_time = time.perf_counter() - self.gauges: Dict[str, GaugeNode] = {} - self.metadata: Dict[str, str] = {} - self._add_default_metadata() - - def push(self, name: str) -> TimerNode: - """ - Called when entering a new block of code that is timed (e.g. with a contextmanager). - """ - current_node: TimerNode = self.stack[-1] - next_node = current_node.get_child(name) - self.stack.append(next_node) - return next_node - - def pop(self) -> None: - """ - Called when exiting a new block of code that is timed (e.g. with a contextmanager). - """ - self.stack.pop() - - def get_root(self) -> TimerNode: - """ - Update the total time and count of the root name, and return it. - """ - root = self.root - root.total = time.perf_counter() - self.start_time - root.count = 1 - return root - - def get_timing_tree(self, node: TimerNode = None) -> Dict[str, Any]: - """ - Recursively build a tree of timings, suitable for output/archiving. - """ - res: Dict[str, Any] = {} - if node is None: - # Special case the root - total is time since it was created, and count is 1 - node = self.get_root() - res["name"] = "root" - - # Only output gauges at top level - if self.gauges: - res["gauges"] = self._get_gauges() - - if self.metadata: - self.metadata["end_time_seconds"] = str(int(time.time())) - res["metadata"] = self.metadata - - res["total"] = node.total - res["count"] = node.count - - if node.is_parallel: - # Note when the block ran in parallel, so that it's less confusing that a timer is less that its children. - res["is_parallel"] = True - - child_total = 0.0 - child_dict = {} - for child_name, child_node in node.children.items(): - child_res: Dict[str, Any] = self.get_timing_tree(child_node) - child_dict[child_name] = child_res - child_total += child_res["total"] - - # "self" time is total time minus all time spent on children - res["self"] = max(0.0, node.total - child_total) - if child_dict: - res["children"] = child_dict - - return res - - def set_gauge(self, name: str, value: float) -> None: - if math.isnan(value): - return - gauge_node = self.gauges.get(name) - if gauge_node: - gauge_node.update(value) - else: - self.gauges[name] = GaugeNode(value) - - def add_metadata(self, key: str, value: str) -> None: - self.metadata[key] = value - - def _get_gauges(self) -> Dict[str, Dict[str, float]]: - gauges = {} - for gauge_name, gauge_node in self.gauges.items(): - gauges[gauge_name] = gauge_node.as_dict() - return gauges - - def _add_default_metadata(self): - self.metadata["timer_format_version"] = TIMER_FORMAT_VERSION - self.metadata["start_time_seconds"] = str(int(time.time())) - self.metadata["python_version"] = sys.version - self.metadata["command_line_arguments"] = " ".join(sys.argv) - - -# Maintain a separate "global" timer per thread, so that they don't accidentally conflict with each other. -_thread_timer_stacks: Dict[int, TimerStack] = {} - - -def _get_thread_timer() -> TimerStack: - ident = threading.get_ident() - if ident not in _thread_timer_stacks: - timer_stack = TimerStack() - _thread_timer_stacks[ident] = timer_stack - return _thread_timer_stacks[ident] - - -def get_timer_stack_for_thread(t: threading.Thread) -> Optional[TimerStack]: - if t.ident is None: - # Thread hasn't started, shouldn't ever happen - return None - return _thread_timer_stacks.get(t.ident) - - -@contextmanager -def hierarchical_timer(name: str, timer_stack: TimerStack = None) -> Generator: - """ - Creates a scoped timer around a block of code. This time spent will automatically be incremented when - the context manager exits. - """ - timer_stack = timer_stack or _get_thread_timer() - timer_node = timer_stack.push(name) - start_time = time.perf_counter() - - try: - # The wrapped code block will run here. - yield timer_node - finally: - # This will trigger either when the context manager exits, or an exception is raised. - # We'll accumulate the time, and the exception (if any) gets raised automatically. - elapsed = time.perf_counter() - start_time - timer_node.add_time(elapsed) - timer_stack.pop() - - -# This is used to ensure the signature of the decorated function is preserved -# See also https://github.com/python/mypy/issues/3157 -FuncT = TypeVar("FuncT", bound=Callable[..., Any]) - - -def timed(func: FuncT) -> FuncT: - """ - Decorator for timing a function or method. The name of the timer will be the qualified name of the function. - Usage: - @timed - def my_func(x, y): - return x + y - Note that because this doesn't take arguments, the global timer stack is always used. - """ - - def wrapped(*args, **kwargs): - with hierarchical_timer(func.__qualname__): - return func(*args, **kwargs) - - return wrapped # type: ignore - - -def set_gauge(name: str, value: float, timer_stack: TimerStack = None) -> None: - """ - Updates the value of the gauge (or creates it if it hasn't been set before). - """ - timer_stack = timer_stack or _get_thread_timer() - timer_stack.set_gauge(name, value) - - -def merge_gauges(gauges: Dict[str, GaugeNode], timer_stack: TimerStack = None) -> None: - """ - Merge the gauges from another TimerStack with the provided one (or the - current thread's stack if none is provided). - :param gauges: - :param timer_stack: - :return: - """ - timer_stack = timer_stack or _get_thread_timer() - for n, g in gauges.items(): - if n in timer_stack.gauges: - timer_stack.gauges[n].merge(g) - else: - timer_stack.gauges[n] = g - - -def add_metadata(key: str, value: str, timer_stack: TimerStack = None) -> None: - timer_stack = timer_stack or _get_thread_timer() - timer_stack.add_metadata(key, value) - - -def get_timer_tree(timer_stack: TimerStack = None) -> Dict[str, Any]: - """ - Return the tree of timings from the TimerStack as a dictionary (or the - current thread's stack if none is provided) - """ - timer_stack = timer_stack or _get_thread_timer() - return timer_stack.get_timing_tree() - - -def get_timer_root(timer_stack: TimerStack = None) -> TimerNode: - """ - Get the root TimerNode of the timer_stack (or the current thread's - TimerStack if not specified) - """ - timer_stack = timer_stack or _get_thread_timer() - return timer_stack.get_root() - - -def reset_timers(timer_stack: TimerStack = None) -> None: - """ - Reset the timer_stack (or the current thread's TimerStack if not specified) - """ - timer_stack = timer_stack or _get_thread_timer() - timer_stack.reset() diff --git a/3DOrangeTree/moduleConspec.py b/3DOrangeTree/moduleConspec.py deleted file mode 100644 index bc2491e..0000000 --- a/3DOrangeTree/moduleConspec.py +++ /dev/null @@ -1,81 +0,0 @@ -from misc_util import orthogonal_init, xavier_uniform_init -import torch.nn as nn -import torch -import torch.nn.functional as F -import math -import numpy as np -import matplotlib.pyplot as plt -from a2c_ppo_acktr.utils import init -import traceback - -np.set_printoptions(threshold=10_000) -torch.set_printoptions(threshold=10_000) - - -class moduleCL(nn.Module): - def __init__(self, - input_size, hidden_size, head, device, args): - super(moduleCL, self).__init__() - self.head = head - self.input_size = input_size - self.hidden_size = hidden_size - self.layers = [None] * head - self.device = device - self.args = args - - - self.main = nn.ModuleList([nn.Sequential( - (nn.Linear(in_features=input_size, out_features=hidden_size, bias=False)), nn.ReLU(), - ) for i in range(head)]) - self.layers = nn.ModuleList([nn.Sequential( - (nn.Linear(in_features=hidden_size, out_features=hidden_size,bias=False)), nn.ReLU(), - ) for i in range(head)]) - - self.layers2 = nn.ParameterList([nn.Parameter( - torch.randn(1, self.hidden_size) * 1.) for i - in range(head)]) - - self.cos = nn.CosineSimilarity(dim=2, eps=1e-6) - - def forward(self, hidden, reward, keyused, obs, time, seed, doimage): - keysUsedOrig = keyused - out1 = [None] * self.head - cossim2 = [None] * self.head - ortho2 = [None] * self.head - minibatchsize = hidden.shape[1] - miniSuccesssize = int(hidden.shape[1] / 2) - - for iii in range(self.head): - out1[iii] = self.layers[iii](self.main[iii](hidden)) - s1, s2, s3 = out1[iii].shape - currentweights = self.layers2[iii].reshape(1, 1, -1).repeat(s1, 1, 1) - cossim2[iii] = self.cos(out1[iii], currentweights) - ortho2[iii] = F.softmax(cossim2[iii].squeeze() * 100., dim=0) - - cossimtotal = torch.stack(cossim2, dim=2) - orthototal = torch.stack(ortho2, dim=2) - cossimtotalmaxxx, indmaxes = ( torch.max(cossimtotal, dim=0)) # 16,3 - cossimtotalmax = cossimtotalmaxxx.squeeze() # minibatch,keys - - # ''' - costFit = ((torch.abs(1 - cossimtotalmax[:miniSuccesssize]).mean(0) + torch.abs( - cossimtotalmax[miniSuccesssize:]).mean(0)).squeeze()) * 1. - if keyused > -0.5: - costFitFinal = costFit[keysUsedOrig] - else: - costFitFinal = costFit.sum() - orthototalperm = orthototal.permute(1, 0, 2) - orthototalperm = F.normalize(orthototalperm, dim=1, p=2) - - - cosnorm = torch.abs(torch.matmul(orthototalperm.permute(0, 2, 1), orthototalperm)) - cosnormdiag = torch.diagonal(cosnorm, dim1=1, dim2=2) - orthogonality = (cosnorm[:, :, :])[:miniSuccesssize] - torch.diag_embed(cosnormdiag[:, :], dim1=1, dim2=2)[ - :miniSuccesssize] - orthogonalitycost = orthogonality.sum() - costfinal = costFitFinal * 1. + (orthogonalitycost * .2 ) /self.head - pos = torch.abs(cossimtotalmax[:miniSuccesssize]).mean(0) - neg = torch.abs(cossimtotalmax[miniSuccesssize:]).mean(0) - - return costfinal, cossimtotal, [pos, neg] - diff --git a/3DOrangeTree/requirements.txt b/3DOrangeTree/requirements.txt deleted file mode 100644 index 13ca8b3..0000000 --- a/3DOrangeTree/requirements.txt +++ /dev/null @@ -1,75 +0,0 @@ -absl-py==1.0.0 -astor==0.8.1 -atari-py==0.1.15 -cached-property==1.5.2 -cachetools==5.0.0 -certifi==2021.10.8 -charset-normalizer==2.0.12 -cloudpickle==1.2.2 -contextlib2==21.6.0 -cycler==0.11.0 -decorator==4.4.2 -dm-sonnet==1.34 -dm-tree==0.1.7 -fonttools==4.33.3 -gast==0.5.3 -google-api-core==2.7.3 -google-api-python-client==2.46.0 -google-auth==2.6.6 -google-auth-httplib2==0.1.0 -googleapis-common-protos==1.56.0 -grpcio==1.44.0 -grpcio-tools==1.44.0 -gym==0.12.6 -gym-notices==0.0.6 -h5py==3.6.0 -httplib2==0.20.4 -idna==3.3 -imageio==2.19.0 -imageio-ffmpeg==0.4.7 -importlib-metadata==4.11.3 -Keras-Applications==1.0.8 -Keras-Preprocessing==1.1.2 -kiwisolver==1.4.2 -kornia==0.6.6 -Markdown==3.3.7 -matplotlib==3.5.1 -mock==4.0.3 -moviepy==1.0.3 -numpy==1.21.6 -opencv-python==4.5.5.64 -packaging==21.3 -pandas==1.3.5 -Pillow==9.1.0 -plotly==5.9.0 -proglog==0.1.9 -protobuf==3.20.1 -psutil==5.9.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pybullet==3.2.4 -pycolab==1.2 -pyglet==1.5.23 -pyparsing==3.0.8 -python-dateutil==2.8.2 -pytz==2022.1 -requests==2.27.1 -rsa==4.8 -scipy==1.7.3 -semantic-version==2.9.0 -six==1.16.0 -stable-baselines3==0.9.0 -tenacity==8.0.1 -tensorboard==1.13.1 -termcolor==1.1.0 -torch==1.10.1+cu111 -torchaudio==0.10.1+rocm4.1 -torchvision==0.11.2+cu111 -tqdm==4.64.0 -trfl==1.2.0 -typing_extensions==4.2.0 -uritemplate==4.1.1 -urllib3==1.26.9 -Werkzeug==2.1.2 -wrapt==1.14.1 -zipp==3.8.0 diff --git a/conspecfunction/Conspec/ConSpec.py b/Conspec/ConSpec.py similarity index 68% rename from conspecfunction/Conspec/ConSpec.py rename to Conspec/ConSpec.py index 226083b..a9951b4 100644 --- a/conspecfunction/Conspec/ConSpec.py +++ b/Conspec/ConSpec.py @@ -1,12 +1,13 @@ -import numpy +import numpy as np import torch import torch.nn.functional as F import torch.nn as nn from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler import torch.optim as optim from .storageConSpec import RolloutStorage -from .loss import prototypes +from .prototype import prototypes from .modelConSpec import EncoderConSpec +import matplotlib.pyplot as plt @@ -60,28 +61,39 @@ def store_memories(self,image_to_store, memory_to_store, action_to_store, reward self.rollouts.insert_trajectory_batch(image_to_store, memory_to_store, action_to_store, reward_to_store, masks_to_store) # self.rollouts.to(self.device) self.rollouts.addPosNeg('pos', self.device) ###add to the positive memory buffer - self.rollouts.addPosNeg('neg', self.device)###add to the negative memory buffer + self.rollouts.addPosNeg('neg', self.device) ###add to the negative memory buffer def calc_cos_scores(self,obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, prototype_number): '''computes the cosine similarity scores''' - hidden = self.encoder.retrieve_hiddens(obs_batch, recurrent_hidden_states_batch,masks_batch, actions_batch) + #print(obs_batch.shape) -> torch.Size([2000, 3, 5, 5]) + hidden = self.encoder.retrieve_hiddens(obs_batch, recurrent_hidden_states_batch, masks_batch) + #print('in cos scores in Conspec', hidden.shape) # hidden.shape after encoder is torch.Size([2000, 512]) + #print('in cose scores in Conspec', hidden.view(*obs_batchorig.size()[:2], -1).shape, obs_batchorig.shape) + # hidden.view(*obs_batchorig.size()[:2], -1).shape = torch.Size([125, 16, 512]) + # obs_batchorig.shape = torch.Size([125, 16, 3, 5, 5]) + #print('prototype number in cos scores in Conspec', prototype_number) => -1 return self.prototypes(hidden.view(*obs_batchorig.size()[:2], -1), prototype_number) def calc_intrinsic_reward(self): '''computes the intrinsic reward for the current minibatch of trajectories''' prototypes_used, count_prototypes_timesteps_criterion = self.rollouts.retrieve_prototypes_used() prototypes_used = prototypes_used.to(device=self.device) + #print('calc intrisic reward proto used',prototypes_used ) with torch.no_grad(): obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = self.rollouts.retrieve_batch() - _, cos_scores, _ = self.calc_cos_scores(obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, -1) + _, _, _, cos_scores, _ = self.calc_cos_scores(obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, -1) + #print('in calc intrisic cos scores shape before reshape', cos_scores.shape) # torch.Size([125, 16, 8]) 10 is the number of prototypes cos_scores = cos_scores.view(*obs_batchorig.size()[:2], -1) # length, minibatch, keyhead + #print('cos score after reshape:', cos_scores.shape) # torch.Size([125, 32, 8]) cos_scores = ((cos_scores > 0.6)) * cos_scores - prototypes_used = torch.tile(torch.reshape(prototypes_used, (1, 1, -1)), (*cos_scores.shape[:2], 1)) + prototypes_used = torch.tile(torch.reshape(prototypes_used, (1, 1, -1)), (*cos_scores.shape[:2], 1)) # + prototypes_used = prototypes_used * self.intrinsicR_scale + intrinsic_reward = (cos_scores * prototypes_used) roundhalf = 3 - '''find the max rewrads in each rollng average (equation 2 of the manuscript)''' + '''find the max rewrads in each rolling average (equation 2 of the manuscript)''' rolling_max = [] for i in range(roundhalf * 2 + 1): temp = torch.roll(intrinsic_reward, i - roundhalf, dims=0) @@ -107,29 +119,49 @@ def calc_intrinsic_reward(self): def update_conspec(self): '''trains the ConSpec module''' prototypes_used, count_prototypes_timesteps_criterion = self.rollouts.retrieve_prototypes_used() - wwtotalpos = [] - wwtotalneg = [] - attentionCL = [] + # wwtotalpos = [] + # wwtotalneg = [] + # attentionCL = [] + cos_scores_pos = [] + cos_scores_neg = [] costCL = 0 if self.rollouts.stepS > self.rollouts.success - 1: ######################## for j in range(self.num_prototypes): if prototypes_used[j] > 0.5: + print('prototypes used', j, prototypes_used[j]) obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = self.rollouts.retrieve_SFbuffer_frozen( j) else: + print('still using SF buffer not the frozen one') obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = self.rollouts.retrieve_SFbuffer() - costCL0, attentionCL0, ww = self.calc_cos_scores(obs_batch, recurrent_hidden_states_batch, masks_batch, - actions_batch, obs_batchorig, j) - cossimtotalmaxxx, _ = (torch.max(attentionCL0, dim=0)) - attentionCL.append(attentionCL0[:, :, j].squeeze().transpose(1, 0)) - costCL += costCL0 - wwtotalpos.append(ww[0][j].detach().cpu()) - wwtotalneg.append(ww[1][j].detach().cpu()) + cos_max_score, max_inds, cost_prototype, cos_scores, cos_score_sf = self.calc_cos_scores(obs_batch, recurrent_hidden_states_batch, masks_batch, + actions_batch, obs_batchorig, j) + # self.rollouts.cos_scores = cos_scores # torch.Size([125, 16, 8]) + # _, costCL0, attentionCL0, ww = self.calc_cos_scores(obs_batch, recurrent_hidden_states_batch, masks_batch, + # actions_batch, obs_batchorig, j) + + # cossimtotalmaxxx, _ = (torch.max(attentionCL0, dim=0)) + # attentionCL.append(attentionCL0[:, :, j].squeeze().transpose(1, 0)) + # costCL += costCL0 + # wwtotalpos.append(ww[0][j].detach().cpu()) + # wwtotalneg.append(ww[1][j].detach().cpu()) + + costCL += cost_prototype + cos_scores_pos.append(cos_score_sf[0][j].detach().cpu()) # torch.Size([16, 8]) + cos_scores_neg.append(cos_score_sf[1][j].detach().cpu()) # torch.Size([16, 8]) + self.rollouts.cos_max_scores = cos_max_score + self.rollouts.max_indx = max_inds + self.rollouts.cos_scores = cos_scores + # self.rollouts.cos_score_pos = cos_score_sf[0] + # self.rollouts.cos_score_neg = cos_score_sf[1] for i in range(self.num_prototypes): - if (wwtotalpos[i] - wwtotalneg[i] > 0.6) and wwtotalpos[i] > 0.6: + # if (wwtotalpos[i] - wwtotalneg[i] > 0.6) and wwtotalpos[i] > 0.6: + #print('hereeee', cos_scores_pos[i], cos_scores_neg[i], cos_scores_pos[i] - cos_scores_neg[i]) + + if (cos_scores_pos[i] - cos_scores_neg[i] > 0.6) and cos_scores_pos[i] > 0.6: count_prototypes_timesteps_criterion[i] += 1 else: count_prototypes_timesteps_criterion[i] = 0 @@ -148,3 +180,5 @@ def do_everything(self, obstotal, recurrent_hidden_statestotal, actiontotal,rew rewardtotal_intrisic_extrinsic = self.calc_intrinsic_reward() self.update_conspec() return rewardtotal_intrisic_extrinsic + + diff --git a/conspecfunction/Conspec/modelConSpec.py b/Conspec/modelConSpec.py similarity index 99% rename from conspecfunction/Conspec/modelConSpec.py rename to Conspec/modelConSpec.py index 9836705..310635e 100644 --- a/conspecfunction/Conspec/modelConSpec.py +++ b/Conspec/modelConSpec.py @@ -72,7 +72,7 @@ def evaluate_actions(self, inputs, rnn_hxs, masks, action): return value, action_log_probs, dist_entropy, rnn_hxs - def retrieve_hiddens(self, inputs, rnn_hxs, masks, action): + def retrieve_hiddens(self, inputs, rnn_hxs, masks): value, actor_features, rnn_hxs = self.base(inputs, rnn_hxs, masks) return actor_features @@ -178,7 +178,6 @@ def __init__(self, num_inputs, recurrent=False, hidden_size=512): constant_(x, 0)) self.critic_linear = init_(nn.Linear(hidden_size, 1)) - self.train() def forward(self, inputs, rnn_hxs, masks): diff --git a/conspecfunction/Conspec/ppoConSpec.py b/Conspec/ppoConSpec.py similarity index 100% rename from conspecfunction/Conspec/ppoConSpec.py rename to Conspec/ppoConSpec.py diff --git a/conspecfunction/Conspec/loss.py b/Conspec/prototype.py similarity index 63% rename from conspecfunction/Conspec/loss.py rename to Conspec/prototype.py index 56f4ae8..ad30066 100644 --- a/conspecfunction/Conspec/loss.py +++ b/Conspec/prototype.py @@ -26,12 +26,12 @@ def __init__(self, (nn.Linear(in_features=hidden_size, out_features=hidden_size,bias=False)), nn.ReLU(), ) for i in range(num_prototypes)]) - self.prototypes = nn.ParameterList([nn.Parameter( - torch.randn(1, self.hidden_size) * 1.) for i - in range(num_prototypes)]) + self.prototypes = nn.ParameterList([nn.Parameter(torch.randn(1, self.hidden_size) * 1.) for i in range(num_prototypes)]) self.cos = nn.CosineSimilarity(dim=2, eps=1e-6) - def forward(self, hidden, prototype_train): # obs = [600, 16, 1, 54, 64 + def forward(self, hidden, prototype_train): # obs = [600, 16, 1, 54, 64] + #print('all prototypes: ', len(self.prototypes)) # 8 + # print('hidden shape in forward prototype = ', hidden.shape) # hidden.shape = torch.Size([125, 16, 512]) out1 = [None] * self.num_prototypes cos_scores = [None] * self.num_prototypes ortho = [None] * self.num_prototypes @@ -39,19 +39,25 @@ def forward(self, hidden, prototype_train): # obs = [600, 16, 1, 54, 64 for i in range(self.num_prototypes): out1[i] = self.layers2[i](self.layers1[i](hidden)) # hidden = length, minibatch, latent s1, s2, s3 = out1[i].shape + #print('prototype shape=', self.prototypes[i].shape) # self.prototypes[i].shape = torch.Size([1, 1010])) prototypes = self.prototypes[i].reshape(1, 1, -1).repeat(s1, 1, 1) + #print('in forward fun of prototype class: ', out1[i].shape, prototypes.shape) # out1[i].shape = torch.Size([125, 16, 1010]), prototypes.shape = torch.Size([125, 1, 1010]) cos_scores[i] = self.cos(out1[i], prototypes) + #print('cos scores in prototype forward class: ', cos_scores[i].shape) # cos scores[i]: torch.Size([125, 16]) ortho[i] = F.softmax(cos_scores[i].squeeze() * 100., dim=0) # length,minibatch,1 --> length,minibatch - cos_scores = torch.stack(cos_scores, dim=2) + cos_scores = torch.stack(cos_scores, dim=2) # stacked cos scores: torch.Size([125, 16, 8]) + #print('stacked cos scores:', cos_scores.shape) ortho_scores = torch.stack(ortho, dim=2) # length,minibatch,keys - cos_max, _ = (torch.max(cos_scores, dim=0)) - cos_max = cos_max.squeeze() # minibatch,keys + cos_max, indices = torch.max(cos_scores, dim=0) # cos_max: torch.Size([16, 8]) + # cos_max = cos_max.squeeze() # minibatch,keys loss_cos = ((torch.abs(1 - cos_max[:success_inds]).mean(0) + torch.abs( cos_max[success_inds:]).mean(0)).squeeze()) + #print('prototype train: ', prototype_train) = -1 if prototype_train > -0.5: loss_cos = loss_cos[prototype_train] else: loss_cos = loss_cos.sum() + #print('loss_cos: ', loss_cos) ortho_scores = F.normalize(ortho_scores.permute(1, 0, 2), dim=1, p=2) ortho_scores = torch.abs(torch.matmul(ortho_scores.permute(0, 2, 1), ortho_scores)) ortho_scores_diag = torch.diagonal(ortho_scores, dim1=1, dim2=2) @@ -61,6 +67,13 @@ def forward(self, hidden, prototype_train): # obs = [600, 16, 1, 54, 64 success_scores = torch.abs(cos_max[:success_inds]).mean(0) fail_scores = torch.abs(cos_max[success_inds:]).mean(0) - return costfinal, cos_scores, [success_scores, fail_scores] + return cos_max, indices, costfinal, cos_scores, [success_scores, fail_scores] + def get_prototype_data(self): + prototypes = {} + for i, param in enumerate(self.prototypes): + # Convert parameter to numpy array and flatten it + param_data = param.detach().cpu().numpy().flatten() # each prototype shape is (1010,) + prototypes['proto '+str(i)] = param_data + return prototypes \ No newline at end of file diff --git a/conspecfunction/Conspec/storageConSpec.py b/Conspec/storageConSpec.py similarity index 86% rename from conspecfunction/Conspec/storageConSpec.py rename to Conspec/storageConSpec.py index 775a7f3..a53dff0 100644 --- a/conspecfunction/Conspec/storageConSpec.py +++ b/Conspec/storageConSpec.py @@ -33,8 +33,10 @@ def __init__(self, num_steps, num_processes, obs_shape, action_space, action_shape = 1 self.actions = torch.zeros(num_steps, num_processes, action_shape) self.actions = self.actions.long() - self.masks = torch.ones(num_steps + 1, num_processes, 1) + self.max_indx = torch.zeros(num_processes, num_prototypes) # torch.Size([32, 8]) + self.cos_scores = torch.zeros(num_steps, num_processes, num_prototypes) # torch.Size([125, 32, 8]) + self.cos_max_scores= torch.zeros(num_processes, num_prototypes) # torch.Size([32, 8]) # Masks that indicate whether it's a true terminal state # or time limit end state @@ -50,7 +52,6 @@ def __init__(self, num_steps, num_processes, obs_shape, action_space, self.success_sample = 16 self.hidden_state_size = 256 - self.obs_batchS = torch.zeros(self.num_steps + 1, self.success, *self.obs_shape) self.r_batchS = torch.zeros(self.num_steps, self.success, 1) self.recurrent_hidden_statesS = torch.zeros(self.num_steps + 1, self.success, self.recurrent_hidden_state_size) @@ -58,6 +59,7 @@ def __init__(self, num_steps, num_processes, obs_shape, action_space, self.act_batchS = self.act_batchS.long() self.masks_batchS = torch.zeros(self.num_steps + 1, self.success, 1) self.stepS = 0 + self.obs_batchF = torch.zeros(self.num_steps + 1, self.success, *self.obs_shape) self.r_batchF = torch.zeros(self.num_steps, self.success, 1) self.recurrent_hidden_statesF = torch.zeros(self.num_steps + 1, self.success, self.recurrent_hidden_state_size) @@ -67,7 +69,6 @@ def __init__(self, num_steps, num_processes, obs_shape, action_space, self.stepF = 0 - self.obs_batch_frozen_S = [None] * self.num_prototypes self.r_batch_frozen_S = [None] * self.num_prototypes self.recurrent_hidden_statesbatch_frozen_S = [None] * self.num_prototypes @@ -99,6 +100,7 @@ def __init__(self, num_steps, num_processes, obs_shape, action_space, self.prototypesUsed = torch.zeros(self.num_prototypes,) self.count_prototypes_timesteps_criterion = torch.zeros(self.num_prototypes,) + def calc_total_reward(self, contrastval): self.rewardsORIG = torch.clone(self.rewards) self.rewards = self.rewards + contrastval.unsqueeze(-1) @@ -128,6 +130,7 @@ def store_frozen_SF(self, i): self.act_batch_frozen_F[i] = self.act_batchF self.masks_batch_frozen_F[i] = self.masks_batchF + def addPosNeg(self, S_or_F, device): '''this is the function for adding the trajectories to success (pos) and failure (neg) memory buffers ''' totalreward = self.rewards[-10:].sum(0) @@ -135,20 +138,25 @@ def addPosNeg(self, S_or_F, device): if S_or_F == 'pos': rewardssortgood = torch.nonzero(totalreward > 0.5).reshape(-1, ) indicesrewardbatch = rewardssortgood[0::2] + # print('pos: indicesrewardbatch', indicesrewardbatch) obsxx = self.obs[:, indicesrewardbatch].to(device) numberaddedxx = obsxx.shape[1] if numberaddedxx >1: indicesrewardbatch = rewardssortgood[0:4:2] + # print('pos: indicesrewardbatch', indicesrewardbatch) else: rewardssortbad = torch.nonzero(totalreward < 0.5).reshape(-1, ) indicesrewardbatch = rewardssortbad[0::2] + # print('neg: indicesrewardbatch', indicesrewardbatch) + obs = self.obs[:, indicesrewardbatch].to(device) rec = self.recurrent_hidden_states[:, indicesrewardbatch].to(device) masks = self.masks[:, indicesrewardbatch].to(device) act = self.actions[:, indicesrewardbatch].to(device) rew = self.rewards[:, indicesrewardbatch].to(device) + # cos = self.cos_scores[:, indicesrewardbatch].to(device) numberadded = obs.shape[1] - + # print('numberadded', numberadded) # print(obs.shape, rec.shape, masks.shape, act.shape, rew.shape) # ''' @@ -158,24 +166,33 @@ def addPosNeg(self, S_or_F, device): elif S_or_F == 'neg': numcareabout = self.stepF if numberadded + numcareabout <= self.obs_batchS.shape[1]: + # print('numberadded + numcareabout <= self.obs_batchS.shape[1]', numberadded, numcareabout, self.obs_batchS.shape[1]) if S_or_F == 'pos': self.obs_batchS[:, self.stepS:self.stepS + numberadded] = obs self.r_batchS[:, self.stepS:self.stepS + numberadded] = rew self.recurrent_hidden_statesS[:, self.stepS:self.stepS + numberadded] = rec self.act_batchS[:, self.stepS:self.stepS + numberadded] = act self.masks_batchS[:, self.stepS:self.stepS + numberadded] = masks + # print('add pos, stepS, stepS+numberadded, numberadded:', self.stepS, self.stepS + numberadded, numberadded) + # print(self.cos_score_pos.shape, cos.shape) + # self.cos_score_pos[:, self.stepS:self.stepS + numberadded] = cos self.stepS = (self.stepS + numberadded) + elif S_or_F == 'neg': self.obs_batchF[:, self.stepF:self.stepF + numberadded] = obs self.r_batchF[:, self.stepF:self.stepF + numberadded] = rew self.recurrent_hidden_statesF[:, self.stepF:self.stepF + numberadded] = rec self.act_batchF[:, self.stepF:self.stepF + numberadded] = act self.masks_batchF[:, self.stepF:self.stepF + numberadded] = masks + # print('add pos, stepF, stepF+numbertoadd, numbertoadd:', self.stepF, self.stepF + numbertoadd, numbertoadd) + # print(self.cos_score_neg.shape, cos.shape) + # self.cos_score_neg[:, self.stepF:self.stepF + numberadded] = cos self.stepF = (self.stepF + numberadded) #''' elif (numberadded + numcareabout >= self.obs_batchS.shape[1]) and ( numcareabout < self.obs_batchS.shape[1]): - + # print('numberadded + numcareabout >= self.obs_batchS.shape[1]', numberadded, numcareabout, self.obs_batchS.shape[1]) + if S_or_F == 'pos': numbertoadd = self.obs_batchS.shape[1] - self.stepS self.obs_batchS[:, self.stepS:self.stepS + numbertoadd, :] = obs[:, :numbertoadd] @@ -183,6 +200,9 @@ def addPosNeg(self, S_or_F, device): self.recurrent_hidden_statesS[:, self.stepS:self.stepS + numbertoadd] = rec[:, :numbertoadd] self.act_batchS[:, self.stepS:self.stepS + numbertoadd] = act[:, :numbertoadd] self.masks_batchS[:, self.stepS:self.stepS + numbertoadd] = masks[:, :numbertoadd] + # print('add pos:', self.stepS, self.stepS + numbertoadd, numbertoadd) + # print(self.cos_score_pos.shape, cos.shape) + # self.cos_score_pos[:, self.stepS:self.stepS + numbertoadd] = cos[:, :numbertoadd] self.stepS = (self.stepS + numbertoadd) elif S_or_F == 'neg': @@ -192,38 +212,51 @@ def addPosNeg(self, S_or_F, device): self.recurrent_hidden_statesF[:, self.stepF:self.stepF + numbertoadd] = rec[:, :numbertoadd] self.act_batchF[:, self.stepF:self.stepF + numbertoadd] = act[:, :numbertoadd] self.masks_batchF[:, self.stepF:self.stepF + numbertoadd] = masks[:, :numbertoadd] + # print('add neg:', self.stepF, self.stepF + numbertoadd, numbertoadd) + # print(self.cos_score_pos.shape, cos.shape) + # self.cos_score_neg[:, self.stepF:self.stepF + numbertoadd] = cos[:, :numbertoadd] self.stepF = (self.stepF + numbertoadd) - elif numcareabout == self.obs_batchS.shape[ - 1]: + elif numcareabout == self.obs_batchS.shape[1]: + # print('numcareabout == self.obs_batchS.shape[1]', numcareabout, self.obs_batchS.shape[1]) hidden_state = rec masks = masks if S_or_F == 'pos': lenconsider = obs.shape[1] - self.obs_batchS = torch.cat((self.obs_batchS , obs),1) - self.r_batchS = torch.cat((self.r_batchS , rew),1) - self.recurrent_hidden_statesS = torch.cat((self.recurrent_hidden_statesS , rec),1) - self.act_batchS = torch.cat((self.act_batchS , act),1) - self.masks_batchS = torch.cat((self.masks_batchS , masks),1) + self.obs_batchS = torch.cat((self.obs_batchS , obs),dim=1) + self.r_batchS = torch.cat((self.r_batchS , rew),dim=1) + self.recurrent_hidden_statesS = torch.cat((self.recurrent_hidden_statesS , rec),dim=1) + self.act_batchS = torch.cat((self.act_batchS , act),dim=1) + self.masks_batchS = torch.cat((self.masks_batchS , masks),dim=1) + # print('add pos:', lenconsider) + # print(self.cos_score_pos.shape, cos.shape) + # self.cos_score_pos = torch.cat((self.cos_score_pos, cos), dim=1) + self.obs_batchS = self.obs_batchS[:,lenconsider:] self.r_batchS = self.r_batchS[:,lenconsider:] self.recurrent_hidden_statesS = self.recurrent_hidden_statesS[:,lenconsider:] self.act_batchS = self.act_batchS[:,lenconsider:] self.masks_batchS = self.masks_batchS[:,lenconsider:] + # self.cos_score_pos = self.cos_score_pos[:,lenconsider:] elif S_or_F == 'neg': lenconsider = obs.shape[1] - self.obs_batchF = torch.cat((self.obs_batchF, obs), 1) - self.r_batchF = torch.cat((self.r_batchF, rew), 1) - self.recurrent_hidden_statesF = torch.cat((self.recurrent_hidden_statesF, rec),1) - self.act_batchF = torch.cat((self.act_batchF, act),1) - self.masks_batchF = torch.cat((self.masks_batchF, masks), 1) + self.obs_batchF = torch.cat((self.obs_batchF, obs), dim=1) + self.r_batchF = torch.cat((self.r_batchF, rew), dim=1) + self.recurrent_hidden_statesF = torch.cat((self.recurrent_hidden_statesF, rec),dim=1) + self.act_batchF = torch.cat((self.act_batchF, act),dim=1) + self.masks_batchF = torch.cat((self.masks_batchF, masks), dim=1) + # print('add neg:', lenconsider) + # print(self.cos_score_neg.shape, cos.shape) + # self.cos_score_neg = torch.cat((self.cos_score_neg, cos), dim=1) + self.obs_batchF = self.obs_batchF[:, lenconsider:] self.r_batchF = self.r_batchF[:, lenconsider:] self.recurrent_hidden_statesF = self.recurrent_hidden_statesF[:, lenconsider:] self.act_batchF = self.act_batchF[:, lenconsider:] self.masks_batchF = self.masks_batchF[:, lenconsider:] + # self.cos_score_neg = self.cos_score_neg[:, lenconsider:] def to(self, device): '''just adding cuda to all the memory buffers''' @@ -236,6 +269,7 @@ def to(self, device): self.actions = self.actions.to(device) self.masks = self.masks.to(device) self.bad_masks = self.bad_masks.to(device) + # self.cos_score_pos = self.cos_score_pos.to(device) self.obs_batchS = self.obs_batchS.to(device) self.r_batchS = self.r_batchS.to(device) @@ -247,17 +281,21 @@ def to(self, device): self.recurrent_hidden_statesF = self.recurrent_hidden_statesF.to(device) self.act_batchF = self.act_batchF.to(device) self.masks_batchF = self.masks_batchF.to(device) + # self.cos_score_neg = self.cos_score_neg.to(device) + for i in range(self.num_prototypes): self.obs_batch_frozen_S[i] = self.obs_batch_frozen_S[i].to(device) self.r_batch_frozen_S[i] = self.r_batch_frozen_S[i].to(device) self.recurrent_hidden_statesbatch_frozen_S[i] = self.recurrent_hidden_statesbatch_frozen_S[i].to(device) self.act_batch_frozen_S[i] = self.act_batch_frozen_S[i].to(device) self.masks_batch_frozen_S[i] = self.masks_batch_frozen_S[i].to(device) + self.obs_batch_frozen_F[i] = self.obs_batch_frozen_F[i].to(device) self.r_batch_frozen_F[i] = self.r_batch_frozen_F[i].to(device) self.recurrent_hidden_statesbatch_frozen_F[i] = self.recurrent_hidden_statesbatch_frozen_F[i].to(device) self.act_batch_frozen_F[i] = self.act_batch_frozen_F[i].to(device) self.masks_batch_frozen_F[i] = self.masks_batch_frozen_F[i].to(device) + def insert(self, obs, recurrent_hidden_states, actions, rewards, masks, bad_masks): self.obs[self.step + 1].copy_(obs) diff --git a/3DOrangeTree/LICENSE b/LICENSE similarity index 100% rename from 3DOrangeTree/LICENSE rename to LICENSE diff --git a/README.md b/README.md index 5c39cee..f12ac40 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,55 @@ - # Contrastive Retrospection (ConSpec) ### Chen Sun, Wannan Yang, Thomas Jiralerspong, Dane Malenfant, Benjamin Alsbury-Nealy, Yoshua Bengio, Blake Richards 2023 This repository implements Contrastive Retrospection (ConSpec) as detailed in the paper: https://arxiv.org/pdf/2210.05845.pdf -**The conspecfunction subfolder implements ConSpec as a self-contained Class that can be added onto any RL agent of choice and trained autonomously.** It has documentation, and should be easy to use for interested researchers. **It also contains a working example of ConSpec helping train a 4-key gridworld task as seen in the paper from fig. 4.** +**ConSpec is implemented as a self-contained Class that can be added onto any RL agent of choice and trained autonomously.** It contains a working example of ConSpec helping train a 4-key gridworld task as seen in the paper from fig. 4. + +![conspeccartoon](Diagram.png) +![movie](Movietask.gif) + +## Information +The basic PPO code was taken from the repository: https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail + +The pycolab tasks codebase was taken from the repository: https://github.com/deepmind/deepmind-research/tree/master/tvt + +## Installation +To create the environment install the packages from the requirements.txt file in a python 3.7 environment. + +## Quick Start + +To see ConSpec in action with an underlying PPO agent on a 4-key task (fig. 4 of the paper), run: +```bash +python -u main.py --pycolab_game key_to_door4 +``` + +## Using ConSpec on top of any other RL agent of choice + +ConSpec is designed to be a self-contained function that can be added to any RL agent of choice. +To use this function in any RL backbone, follow these 3 simple steps: + +1. **Load the ConSpec module**: The `Conspec/` folder contains all necessary files. + +2. **Import and initialize ConSpec** in your main training script: +```python +from Conspec.ConSpec import ConSpec +conspecfunction = ConSpec(args, obsspace, env.num_actions, device) +``` + +3. **Add ConSpec to your RL training loop**: +```python +obstotal, rewardtotal, recurrent_hidden_statestotal, actiontotal, maskstotal = rollouts.release() +reward_intrinsic_extrinsic = conspecfunction.do_everything(obstotal, recurrent_hidden_statestotal, actiontotal, rewardtotal, maskstotal) +rollouts.storereward(reward_intrinsic_extrinsic) +``` -The 3DOrangeTree subfolder implements ConSpec in the OrangeTree task described in the paper but **is a much older repository** - will clean up at a later date. +The purpose of these 3 lines is to: +- retrieve the current minibatch of trajectory (including observations, rewards, hidden states, actions, masks) +- "do everything" that ConSpec needs to do internally for training, and output the intrinsic + extrinsic reward for the current minibatch of trajectories +- store this total reward in the memory buffer +**A working example showing these lines of code added into an RL training loop is seen in main.py** -# ![conspeccartoon](Diagram.png) -# ![movie](Movietask.gif) diff --git a/3DOrangeTree/a2c_ppo_acktr/__init__.py b/a2c_ppo_acktr/__init__.py similarity index 100% rename from 3DOrangeTree/a2c_ppo_acktr/__init__.py rename to a2c_ppo_acktr/__init__.py diff --git a/conspecfunction/a2c_ppo_acktr/algo/__init__.py b/a2c_ppo_acktr/algo/__init__.py similarity index 100% rename from conspecfunction/a2c_ppo_acktr/algo/__init__.py rename to a2c_ppo_acktr/algo/__init__.py diff --git a/conspecfunction/a2c_ppo_acktr/algo/a2c_acktr.py b/a2c_ppo_acktr/algo/a2c_acktr.py similarity index 100% rename from conspecfunction/a2c_ppo_acktr/algo/a2c_acktr.py rename to a2c_ppo_acktr/algo/a2c_acktr.py diff --git a/3DOrangeTree/a2c_ppo_acktr/algo/gail.py b/a2c_ppo_acktr/algo/gail.py similarity index 100% rename from 3DOrangeTree/a2c_ppo_acktr/algo/gail.py rename to a2c_ppo_acktr/algo/gail.py diff --git a/conspecfunction/a2c_ppo_acktr/algo/ppo.py b/a2c_ppo_acktr/algo/ppo.py similarity index 100% rename from conspecfunction/a2c_ppo_acktr/algo/ppo.py rename to a2c_ppo_acktr/algo/ppo.py diff --git a/conspecfunction/a2c_ppo_acktr/algo/ppoConSpec.py b/a2c_ppo_acktr/algo/ppoConSpec.py similarity index 100% rename from conspecfunction/a2c_ppo_acktr/algo/ppoConSpec.py rename to a2c_ppo_acktr/algo/ppoConSpec.py diff --git a/conspecfunction/a2c_ppo_acktr/arguments.py b/a2c_ppo_acktr/arguments.py similarity index 89% rename from conspecfunction/a2c_ppo_acktr/arguments.py rename to a2c_ppo_acktr/arguments.py index 4107004..a7663e8 100644 --- a/conspecfunction/a2c_ppo_acktr/arguments.py +++ b/a2c_ppo_acktr/arguments.py @@ -16,7 +16,7 @@ def get_args(): default=8, help='') parser.add_argument( - '--lrConSpec', type=float, default=20e-4, help='learning rate (default: 7e-4)') + '--lrConSpec', type=float, default=2e-3, help='learning rate (default: 7e-4)') parser.add_argument( '--intrinsicR_scale', type=float, @@ -28,8 +28,15 @@ def get_args(): parser.add_argument( '--pycolab_game', - default='key_to_door4', - help='key_to_door4, key_to_door2, key_to_door3, ') + default='key_to_door3', + help='key_to_door4, key_to_door2, key_to_door3') + + parser.add_argument( + '--num_episodes', + type=int, + default=1000, + help='500 for 2 key, 1000 for 3 keys, 2000 for 4 keys') + parser.add_argument( '--pycolab_apple_reward_min', type=float, @@ -143,7 +150,7 @@ def get_args(): parser.add_argument( '--num-processes', type=int, - default=16, + default=8, help='how many training CPU processes to use (default: 16)') parser.add_argument( '--num-steps', @@ -180,11 +187,22 @@ def get_args(): type=int, default=None, help='eval interval, one eval per n updates (default: None)') + + parser.add_argument( + '--start_checkpoint', + type=int, + default=500, + help='checkpoint interval, one eval per n updates (default: None)') + parser.add_argument( + '--checkpoint_interval', + type=int, + default=100, + help='checkpoint interval, one eval per n updates (default: None)') parser.add_argument( '--num-env-steps', type=int, - default=10e8, - help='number of environment steps to train (default: 10e6)') + default=10e6, + help='number of environment steps to train (default: 10e6-8)') parser.add_argument( '--env-name', default='PongNoFrameskip-v4', diff --git a/3DOrangeTree/a2c_ppo_acktr/distributions.py b/a2c_ppo_acktr/distributions.py similarity index 100% rename from 3DOrangeTree/a2c_ppo_acktr/distributions.py rename to a2c_ppo_acktr/distributions.py diff --git a/conspecfunction/a2c_ppo_acktr/envs.py b/a2c_ppo_acktr/envs.py similarity index 98% rename from conspecfunction/a2c_ppo_acktr/envs.py rename to a2c_ppo_acktr/envs.py index 2d9f316..540fa46 100644 --- a/conspecfunction/a2c_ppo_acktr/envs.py +++ b/a2c_ppo_acktr/envs.py @@ -21,15 +21,15 @@ except ImportError: pass -try: - import roboschool -except ImportError: - pass - -try: - import pybullet_envs -except ImportError: - pass +# try: +# import roboschool +# except ImportError: +# pass + +# try: +# import pybullet_envs +# except ImportError: +# pass def make_env(env_id, seed, rank, log_dir, allow_early_resets): diff --git a/conspecfunction/a2c_ppo_acktr/modelRL.py b/a2c_ppo_acktr/modelRL.py similarity index 100% rename from conspecfunction/a2c_ppo_acktr/modelRL.py rename to a2c_ppo_acktr/modelRL.py diff --git a/conspecfunction/a2c_ppo_acktr/storage.py b/a2c_ppo_acktr/storage.py similarity index 99% rename from conspecfunction/a2c_ppo_acktr/storage.py rename to a2c_ppo_acktr/storage.py index 2a85cf1..84e258a 100644 --- a/conspecfunction/a2c_ppo_acktr/storage.py +++ b/a2c_ppo_acktr/storage.py @@ -54,12 +54,12 @@ def retrieveR(self): def retrieveobs(self): return self.obs - def retrieveeverything(self): + def retrieveeverything(self): ### !!! This function doesn't work return torch.cat((self.obs_batchS, self.obs_batchF), dim=1), torch.cat((self.r_batchS, self.r_batchF), dim=1), torch.cat( (self.masks_batchS, self.masks_batchF), dim=1), torch.cat((self.act_batchS, self.act_batchF), dim=1) - def retrieveRS(self): + def retrieveRS(self):## this too! return self.r_batchS, self.r_batchF diff --git a/3DOrangeTree/a2c_ppo_acktr/utils.py b/a2c_ppo_acktr/utils.py similarity index 100% rename from 3DOrangeTree/a2c_ppo_acktr/utils.py rename to a2c_ppo_acktr/utils.py diff --git a/analysis.py b/analysis.py new file mode 100644 index 0000000..b08ca86 --- /dev/null +++ b/analysis.py @@ -0,0 +1,196 @@ +import torch +import os +import numpy as np +import matplotlib.pyplot as plt + +def vis_proto_states(sf_buffer_obs, cos_scores): + ''' + plot the states that are matched with each prototype using cos scores and not max score + num_prototype figures each has num_processes images + sf_buffer_obs -> (185, 32, 3, 5, 5) : time steps, processes, channels, height, width + cos_scores -> (185, 32, 8) + ''' + cos_score_proto = {} + sf_obs_reshaped = np.reshape(sf_buffer_obs, (cos_scores.shape[0], cos_scores.shape[1], *sf_buffer_obs.shape[1:])) + num_prototypes = cos_scores.shape[2] + + # axes = axes.flatten() + fig, axes = plt.subplots(1, 32, figsize=(32 * 2, 2)) + + # Iterate over time steps and processes to find observations with maximum cosine similarity + for prototype in range(num_prototypes): + cos_score_proto[prototype] = cos_scores[:,:,prototype] + indx = np.argmax(cos_score_proto[prototype], axis=0) + scores = np.max(cos_score_proto[prototype], axis=0) + obs_over_time = [] + for i in range(indx.shape[0]): + obs = sf_obs_reshaped[indx[i], i] + obs = np.transpose(obs, (1, 2, 0)) + obs_over_time.append(obs) + + # Iterate over the list of images + for i, image in enumerate(obs_over_time): + # Plot the i-th image + if image.max() > 1.0: # assuming the data should be in [0, 1] range + image = image / 255.0 # Normalize + elif image.max() > 1.0: + image = image.astype(np.uint8) + + axes[i].imshow(image) + axes[i].axis('off') + # set the title of each obs with cosine similarity + title = '' + if scores is not None: + score = scores[i] + title += f"Score: {score:.2f}" + axes[i].set_title(title) + plt.tight_layout() + plt.savefig("states_prototypes{}.png".format(prototype)) + + +def extract_observations(cosine_sim_idx_matrix, observation_matrix): + """ + Given matrices: + cosine_similarity_matrix -> (32, 8) or (8, ) + observation_matrix => (185, 32, 3, 5, 5) or (32, 8, 3, 5, 5) + """ + if cosine_sim_idx_matrix.ndim == 2: # When cosine similarity index matrix is 2D + num_time_steps = cosine_sim_idx_matrix.shape[0] + num_prototypes = cosine_sim_idx_matrix.shape[1] + + extracted_observations = np.zeros((num_time_steps, num_prototypes, *observation_matrix.shape[2:])) + for i in range(cosine_sim_idx_matrix.shape[0]): + for j in range(cosine_sim_idx_matrix.shape[1]): + idx = cosine_sim_idx_matrix[i, j] + observation = observation_matrix[idx, i] # Extract the observation + extracted_observations[i, j] = observation # Assign to the result + + elif cosine_sim_idx_matrix.ndim == 1: # When cosine similarity index matrix is 1D + num_prototypes = cosine_sim_idx_matrix.shape[0] + extracted_observations = np.zeros((num_prototypes, *observation_matrix.shape[2:])) + for j in range(num_prototypes): + idx = cosine_sim_idx_matrix[j] + observation = observation_matrix[idx, j] # Extract the observation + extracted_observations[j] = observation # Assign to the result + + return extracted_observations + + +def plot_observations(observations, scores, path, filename): + """ + Given matrices: + scores -> (32, 8) or (8, ) + observations => (32, 8, 3, 5, 5) or (8, 3, 5, 5) + """ + if scores.ndim == 2: + # Number of rows and columns in the cosine similarity matrix + rows, cols = observations.shape[:2] + print(rows, cols) + fig, axs = plt.subplots(rows, cols, figsize=(cols * 2, rows * 2)) + for i in range(rows): + for j in range(cols): + ax = axs[i, j] + obs = observations[i, j].transpose(1, 2, 0) # Adjust for color channel position + + if obs.max() > 1.0: # assuming the data should be in [0, 1] range + obs = obs / 255.0 # Normalize + # Convert to uint8 if data is of integer type but not uint8 + elif obs.max() > 1.0: + obs = obs.astype(np.uint8) + + + ax.imshow(obs, vmin=0, vmax=1) + ax.axis('off') + # add title with cosine similarity score + title = '' + if scores is not None: + score = scores[i, j] + title += f"Score: {score:.2f}" + ax.set_title(title) + plt.tight_layout() + + elif scores.ndim == 1: + rows = observations.shape[0] + print(rows) + fig, axs = plt.subplots(rows, figsize=(rows * 2, rows)) + + for i in range(rows): + ax = axs[i] + obs = observations[i].transpose(1, 2, 0) # Adjust for color channel position + if obs.max() > 1.0: # assuming the data should be in [0, 1] range + obs = obs / 255.0 # Normalize + # Convert to uint8 if data is of integer type but not uint8 + elif obs.max() > 1.0: + obs = obs.astype(np.uint8) + + ax.imshow(obs, vmin=0, vmax=1) + ax.axis('off') + # add title with cosine similarity score + title = '' + if scores is not None: + score = scores[i] + title += f"Score: {score:.2f}, Prototype: {i}" + ax.set_title(title) + plt.tight_layout() + + # Create folder if it doesn't exist + if not os.path.exists(path): + os.makedirs(path) + + # Save the figure + save_path = os.path.join(path, filename) + plt.savefig(save_path) + plt.close(fig) # Close the plot to free up memory + return save_path + + +def vis_proto_max_state_processes(obs, max_cos_score, indices, num_processes, path, filename): + ''' + num_processes rows of num of prototype images, a figure of 32 x 8 + ''' + dim= int(obs.shape[0]/num_processes) + sf_obs_reshaped = np.reshape(obs, (dim, num_processes, *sf_buffer_obs.shape[1:])) + extracted_obs = extract_observations(indices, sf_obs_reshaped) + saved_file_path = plot_observations(extracted_obs, max_cos_score, path, filename) + +def vis_proto_max_state(obs, max_cos_score, indices, num_processes, path, filename): + dim= int(obs.shape[0]/num_processes) + sf_obs_reshaped = np.reshape(obs, (dim, num_processes, *sf_buffer_obs.shape[1:])) + extracted_obs = extract_observations(indices, sf_obs_reshaped) # (32, 8, 3, 5, 5) + maxx_score = np.max(max_cos_score, axis=0) # max over processes + maxx_index = np.argmax(max_cos_score, axis=0) + max_extracted_obs = extract_observations(maxx_index, extracted_obs) + saved_file_path = plot_observations(max_extracted_obs, maxx_score, path, filename) + +if __name__ == "__main__": + + ### Data loaded from key to door 3 envs + base_directory = '/network/scratch/s/samieima/conspec_train/20240325-152802_key_to_door3_1_10000/' + + data_path = 'conspec_rollouts_epoch_9999.pth' + cos_sim_path = 'cos_sim_epoch_9999.pth' + + data_full_path = os.path.join(base_directory, data_path) + cos_full_path = os.path.join(base_directory, cos_sim_path) + + ## Load the data + sf_buffer_obs = torch.load(data_full_path)['obs'] # (185, 32, 3, 5, 5) + cos_max_scores = torch.load(cos_full_path)['cos_max_scores'] # (32, 8) + cos_max_indices = torch.load(cos_full_path)['max_indices'] # (32, 8) + cos_scores = torch.load(cos_full_path)['cos_scores'] # (185, 32, 8) + + sf_buffer_obs = sf_buffer_obs.detach().cpu().numpy() + cos_max_scores = cos_max_scores.detach().cpu().numpy() + cos_max_indices = cos_max_indices.detach().cpu().numpy() + cos_scores = cos_scores.detach().cpu().numpy() + num_processes = cos_max_scores.shape[0] + num_prototypes = cos_max_scores.shape[1] + + figure_path = "data/key_to_door3_10000/figures/" + filename = "max_states_key_to_door3_10000.png" + vis_proto_max_state_processes(sf_buffer_obs, cos_max_scores, cos_max_indices, num_processes, figure_path, filename) + + # vis_proto_states(sf_buffer_obs, cos_scores) + + filename = "max_proto_1state_key_to_door3_10000.png" + vis_proto_max_state(sf_buffer_obs, cos_max_scores, cos_max_indices, num_processes, figure_path, filename) diff --git a/collect_data.py b/collect_data.py new file mode 100644 index 0000000..9fcd3b8 --- /dev/null +++ b/collect_data.py @@ -0,0 +1,162 @@ +import numpy as np +import os +import torch +import csv +import pandas as pd +import glob +import argparse +import pdb + +def collect_gfn_data_max_states(cos_max_scores, max_indices, num_processes, num_prototypes, path, filename): + ''' + sf_buffer_obs: (185, 32, 3, 5, 5) + cos_max_scores: (32, 8) + max_indices: (32, 8) + num_processes: 8 + path: path to save the data + filename: name of the data + ''' + cos_sim_threshold = 0.6 + data = [] + cos_data = [] + + for process_idx in range(num_processes): + traj_proto, cos_proto = [], [] + proto_active = False + for i in range(num_prototypes): + idx = max_indices[process_idx, i] + cos_sim = cos_max_scores[process_idx, i] + if cos_sim > cos_sim_threshold: + proto_active = True + else: + proto_active = False + traj_proto.append(proto_active) + cos_proto.append(cos_sim) + data.append(traj_proto) + cos_data.append(cos_proto) + + if not os.path.exists(path): + os.makedirs(path) + + # Save the figure + save_path = os.path.join(path, filename) + # Write the data to a CSV file + with open(f'{save_path}_binary_th_{cos_sim_threshold}.csv', 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(['P_' + str(i) for i in range(num_prototypes)]) + writer.writerows(data) + with open(f'{save_path}_cos.csv', 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(cos_data) + + +def collect_gfn_data(exp_name, epoch): + base_directory = "data" + full_path = os.path.join(base_directory, exp_name) + + buffer_path = os.path.join(full_path, 'buffer_epoch_{}.pth'.format(epoch)) + sf_buffer = os.path.join(full_path, 'conspec_rollouts_epoch_{}.pth'.format(epoch)) + observations = torch.load(sf_buffer)['obs'] + # observations = torch.load(buffer_path)['obs'] + cos_path = os.path.join(full_path, 'cos_sim_epoch_{}.pth'.format(epoch)) + cos_sim_info = torch.load(cos_path) + cos_scores_max = cos_sim_info['cos_max_scores'].detach().cpu().numpy() + max_indices = cos_sim_info['max_indices'].detach().cpu().numpy() + cos_scores = cos_sim_info['cos_scores'].detach().cpu().numpy() + print(observations.shape, cos_scores.shape) + # Number of batches + num_batches = cos_scores.shape[1] + cos_sim_threshold = 0.6 + num_prototypes = cos_scores_max.shape[1] + data = [] + cos_data = [] + default = (np.zeros((3,5,5)), -1) + state_prototypes = {0:default, 1:default, 2: default, 3: default, 4: default, 5:default, 6: default, 7:default} # prototype, states, cosine similarity + time_steps = int(observations.shape[0] / num_batches) + print(time_steps, num_batches) + for batch in range(num_batches): + for state in range(time_steps): + traj_proto = [] + cos_proto = [] + proto_active = False + for proto in range(num_prototypes): + cos_sim = cos_scores[state, batch, proto] + if cos_sim > cos_sim_threshold: + proto_active = True + else: + proto_active = False + traj_proto.append(proto_active) + cos_proto.append(cos_sim) + data.append(traj_proto) + cos_data.append(cos_proto) + + gfn_path = 'data/gfn/gfn_data_{}_{}.csv'.format(exp_name,epoch) + cos_path = 'data/gfn/cos_data_{}_{}.csv'.format(exp_name, epoch) + # Write the data to a CSV file + with open(gfn_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(['Prototype_' + str(i) for i in range(num_prototypes)]) + writer.writerows(data) + with open(cos_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(cos_data) + # print(state_prototypes) + return state_prototypes + +def merge_csv_files(base_path, prefix, postfix, output_file): + + # Combine all CSV files into a single DataFrame + if postfix == 'binary': + file_list = glob.glob(base_path + "{}*{}*".format(prefix, postfix)) + elif postfix == 'cos': + file_list = glob.glob(base_path + "{}*{}.csv".format(prefix, postfix)) + num_files = len(file_list) + + files = [f for f in file_list] + + combined_csv = pd.concat([pd.read_csv(file) for file in files], ignore_index=True, sort=False) + print(combined_csv.shape) + # Write the combined DataFrame to a new CSV file + combined_csv.to_csv(output_file, index=False) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Collect data for GFN') + parser.add_argument('--exp_name', type=str, default='20240325-152802_key_to_door3_1_10000', help='Experiment name') + parser.add_argument('--num_episodes', type=int, default=10000, help='Epoch number') + parser.add_argument('--cos_sim_threshold', type=float, default=0.6, help='Cosine similarity threshold') + parser.add_argument('--merge_data', action='store_true', default=True, help='if you are merging the csv files') + parser.add_argument('--output_file', type=str, default='merged_binary_kd3_10k_0.6.csv', help='Output file name') + parser.add_argument('--output_csv_files_prefix', type=str, default='gfn_kd3_10k_', help='Output cosine csv file name') + args = parser.parse_args() + + base_directory = '/network/scratch/s/samieima/conspec_train/20240325-152802_key_to_door3_1_10000/' + csv_path = 'csv_files/' + + if args.merge_data == False: + for i in range(args.num_episodes): + if i == 0: + continue + data_path = f'conspec_rollouts_epoch_{i}.pth' + cos_sim_path = f'cos_sim_epoch_{i}.pth' + data_full_path = os.path.join(base_directory, data_path) + cos_full_path = os.path.join(base_directory, cos_sim_path) + + sf_buffer = torch.load(data_full_path)['obs'] + cos_sim_info = torch.load(cos_full_path) + cos_scores_max = cos_sim_info['cos_max_scores'].detach().cpu().numpy() + max_index = cos_sim_info['max_indices'].detach().cpu().numpy() + num_processes = cos_scores_max.shape[0] + num_prototypes = cos_scores_max.shape[1] + + path = f'{base_directory}{csv_path}' + filename = f'gfn_kd3_10k_{i}' + collect_gfn_data_max_states(cos_scores_max, max_index, num_processes, num_prototypes, path, filename) + + elif args.merge_data == True: + base_path = f'{base_directory}{csv_path}' + # merge_csv_files(base_path, 'gfn_kd3_10k_', 'binary', 'merged_binary_kd3_10k_0.6.csv') + merge_csv_files(base_path, 'gfn_kd3_10k_', 'cos', 'merged_cosine_kd3_10k_0.6.csv') + + diff --git a/conspecfunction/LICENSE b/conspecfunction/LICENSE deleted file mode 100644 index 73fa275..0000000 --- a/conspecfunction/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 Ilya Kostrikov - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/conspecfunction/README.md b/conspecfunction/README.md deleted file mode 100644 index 9b373ed..0000000 --- a/conspecfunction/README.md +++ /dev/null @@ -1,47 +0,0 @@ - - - -# ConSpec - -## Information -The basic PPO code was taken from the repository: https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail - -The pycolab tasks codebase was taken from the repository: https://github.com/deepmind/deepmind-research/tree/master/tvt - -## Installation -To create the environment install the packages from the requirements.txt file in a python 3.7 environment. - - - -### Launching - -To see conspecfunction in action with an underlying PPO agent on a 4-key task (fig. 4 of the paper), run: -``` -python -u main.py --pycolab_game key_to_door4 -``` - -### Using ConSpec on top of any other RL agent of choice - -This function (conspecfunction) is meant to be a self-contained function and can be added to any RL agent of choice. -To use this function in any RL backbone, 3 simple steps need to be done (stated below). **A working example showing these lines of code added into an RL training loop is seen in main.py** - - -1. Load the "Conspec" subfolder into your codebase. This subfolder contains 5 files. - -2. In your main training script, import the following. All the relevant ConSpec functions and objects are contained in this class. -``` -from Conspec.ConSpec import ConSpec -conspecfunction = ConSpec(args, obsspace, env.num_actions, device) -``` - -3. In your RL training loop, simply add the 3 lines below. -``` - obstotal, rewardtotal, recurrent_hidden_statestotal, actiontotal, maskstotal = rollouts.release() - reward_intrinsic_extrinsic = conspecfunction.do_everything(obstotal, recurrent_hidden_statestotal, actiontotal, rewardtotal, maskstotal) - rollouts.storereward(reward_intrinsic_extrinsic) -``` -the purpose of these last 3 lines is to: -i. retrieve the current minibatch of trajectory (including its observations, rewards, hidden states, actions, masks) -ii. "do everything" that ConSpec needs to do internally for training, and output the intrinsic + extrinsic reward for the current minibatch of trajectories -iii. store this total reward in the memory buffer - diff --git a/conspecfunction/a2c_ppo_acktr/__init__.py b/conspecfunction/a2c_ppo_acktr/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/conspecfunction/a2c_ppo_acktr/algo/gail.py b/conspecfunction/a2c_ppo_acktr/algo/gail.py deleted file mode 100644 index aa85380..0000000 --- a/conspecfunction/a2c_ppo_acktr/algo/gail.py +++ /dev/null @@ -1,166 +0,0 @@ -import h5py -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.utils.data -from torch import autograd - -from stable_baselines3.common.running_mean_std import RunningMeanStd - -class Discriminator(nn.Module): - def __init__(self, input_dim, hidden_dim, device): - super(Discriminator, self).__init__() - - self.device = device - - self.trunk = nn.Sequential( - nn.Linear(input_dim, hidden_dim), nn.Tanh(), - nn.Linear(hidden_dim, hidden_dim), nn.Tanh(), - nn.Linear(hidden_dim, 1)).to(device) - - self.trunk.train() - - self.optimizer = torch.optim.Adam(self.trunk.parameters()) - - self.returns = None - self.ret_rms = RunningMeanStd(shape=()) - - def compute_grad_pen(self, - expert_state, - expert_action, - policy_state, - policy_action, - lambda_=10): - alpha = torch.rand(expert_state.size(0), 1) - expert_data = torch.cat([expert_state, expert_action], dim=1) - policy_data = torch.cat([policy_state, policy_action], dim=1) - - alpha = alpha.expand_as(expert_data).to(expert_data.device) - - mixup_data = alpha * expert_data + (1 - alpha) * policy_data - mixup_data.requires_grad = True - - disc = self.trunk(mixup_data) - ones = torch.ones(disc.size()).to(disc.device) - grad = autograd.grad( - outputs=disc, - inputs=mixup_data, - grad_outputs=ones, - create_graph=True, - retain_graph=True, - only_inputs=True)[0] - - grad_pen = lambda_ * (grad.norm(2, dim=1) - 1).pow(2).mean() - return grad_pen - - def update(self, expert_loader, rollouts, obsfilt=None): - self.train() - - policy_data_generator = rollouts.feed_forward_generator( - None, mini_batch_size=expert_loader.batch_size) - - loss = 0 - n = 0 - for expert_batch, policy_batch in zip(expert_loader, - policy_data_generator): - policy_state, policy_action = policy_batch[0], policy_batch[2] - policy_d = self.trunk( - torch.cat([policy_state, policy_action], dim=1)) - - expert_state, expert_action = expert_batch - expert_state = obsfilt(expert_state.numpy(), update=False) - expert_state = torch.FloatTensor(expert_state).to(self.device) - expert_action = expert_action.to(self.device) - expert_d = self.trunk( - torch.cat([expert_state, expert_action], dim=1)) - - expert_loss = F.binary_cross_entropy_with_logits( - expert_d, - torch.ones(expert_d.size()).to(self.device)) - policy_loss = F.binary_cross_entropy_with_logits( - policy_d, - torch.zeros(policy_d.size()).to(self.device)) - - gail_loss = expert_loss + policy_loss - grad_pen = self.compute_grad_pen(expert_state, expert_action, - policy_state, policy_action) - - loss += (gail_loss + grad_pen).item() - n += 1 - - self.optimizer.zero_grad() - (gail_loss + grad_pen).backward() - self.optimizer.step() - return loss / n - - def predict_reward(self, state, action, gamma, masks, update_rms=True): - with torch.no_grad(): - self.eval() - d = self.trunk(torch.cat([state, action], dim=1)) - s = torch.sigmoid(d) - reward = s.log() - (1 - s).log() - if self.returns is None: - self.returns = reward.clone() - - if update_rms: - self.returns = self.returns * masks * gamma + reward - self.ret_rms.update(self.returns.cpu().numpy()) - - return reward / np.sqrt(self.ret_rms.var[0] + 1e-8) - - -class ExpertDataset(torch.utils.data.Dataset): - def __init__(self, file_name, num_trajectories=4, subsample_frequency=20): - all_trajectories = torch.load(file_name) - - perm = torch.randperm(all_trajectories['states'].size(0)) - idx = perm[:num_trajectories] - - self.trajectories = {} - - # See https://github.com/pytorch/pytorch/issues/14886 - # .long() for fixing bug in torch v0.4.1 - start_idx = torch.randint( - 0, subsample_frequency, size=(num_trajectories, )).long() - - for k, v in all_trajectories.items(): - data = v[idx] - - if k != 'lengths': - samples = [] - for i in range(num_trajectories): - samples.append(data[i, start_idx[i]::subsample_frequency]) - self.trajectories[k] = torch.stack(samples) - else: - self.trajectories[k] = data // subsample_frequency - - self.i2traj_idx = {} - self.i2i = {} - - self.length = self.trajectories['lengths'].sum().item() - - traj_idx = 0 - i = 0 - - self.get_idx = [] - - for j in range(self.length): - - while self.trajectories['lengths'][traj_idx].item() <= i: - i -= self.trajectories['lengths'][traj_idx].item() - traj_idx += 1 - - self.get_idx.append((traj_idx, i)) - - i += 1 - - - def __len__(self): - return self.length - - def __getitem__(self, i): - traj_idx, i = self.get_idx[i] - - return self.trajectories['states'][traj_idx][i], self.trajectories[ - 'actions'][traj_idx][i] diff --git a/conspecfunction/a2c_ppo_acktr/distributions.py b/conspecfunction/a2c_ppo_acktr/distributions.py deleted file mode 100644 index a705431..0000000 --- a/conspecfunction/a2c_ppo_acktr/distributions.py +++ /dev/null @@ -1,109 +0,0 @@ -import math - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from a2c_ppo_acktr.utils import AddBias, init - -""" -Modify standard PyTorch distributions so they are compatible with this code. -""" - -# -# Standardize distribution interfaces -# - -# Categorical -class FixedCategorical(torch.distributions.Categorical): - def sample(self): - return super().sample().unsqueeze(-1) - - def log_probs(self, actions): - return ( - super() - .log_prob(actions.squeeze(-1)) - .view(actions.size(0), -1) - .sum(-1) - .unsqueeze(-1) - ) - - def mode(self): - return self.probs.argmax(dim=-1, keepdim=True) - - -# Normal -class FixedNormal(torch.distributions.Normal): - def log_probs(self, actions): - return super().log_prob(actions).sum(-1, keepdim=True) - - def entropy(self): - return super().entropy().sum(-1) - - def mode(self): - return self.mean - - -# Bernoulli -class FixedBernoulli(torch.distributions.Bernoulli): - def log_probs(self, actions): - return super.log_prob(actions).view(actions.size(0), -1).sum(-1).unsqueeze(-1) - - def entropy(self): - return super().entropy().sum(-1) - - def mode(self): - return torch.gt(self.probs, 0.5).float() - - -class Categorical(nn.Module): - def __init__(self, num_inputs, num_outputs): - super(Categorical, self).__init__() - - init_ = lambda m: init( - m, - nn.init.orthogonal_, - lambda x: nn.init.constant_(x, 0), - gain=0.01) - - self.linear = init_(nn.Linear(num_inputs, num_outputs)) - - def forward(self, x): - x = self.linear(x) - return FixedCategorical(logits=x) - - -class DiagGaussian(nn.Module): - def __init__(self, num_inputs, num_outputs): - super(DiagGaussian, self).__init__() - - init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init. - constant_(x, 0)) - - self.fc_mean = init_(nn.Linear(num_inputs, num_outputs)) - self.logstd = AddBias(torch.zeros(num_outputs)) - - def forward(self, x): - action_mean = self.fc_mean(x) - - # An ugly hack for my KFAC implementation. - zeros = torch.zeros(action_mean.size()) - if x.is_cuda: - zeros = zeros.cuda() - - action_logstd = self.logstd(zeros) - return FixedNormal(action_mean, action_logstd.exp()) - - -class Bernoulli(nn.Module): - def __init__(self, num_inputs, num_outputs): - super(Bernoulli, self).__init__() - - init_ = lambda m: init(m, nn.init.orthogonal_, lambda x: nn.init. - constant_(x, 0)) - - self.linear = init_(nn.Linear(num_inputs, num_outputs)) - - def forward(self, x): - x = self.linear(x) - return FixedBernoulli(logits=x) diff --git a/conspecfunction/a2c_ppo_acktr/utils.py b/conspecfunction/a2c_ppo_acktr/utils.py deleted file mode 100644 index 328d3a6..0000000 --- a/conspecfunction/a2c_ppo_acktr/utils.py +++ /dev/null @@ -1,65 +0,0 @@ -import glob -import os - -import torch -import torch.nn as nn - -from a2c_ppo_acktr.envs import VecNormalize - - -# Get a render function -def get_render_func(venv): - if hasattr(venv, 'envs'): - return venv.envs[0].render - elif hasattr(venv, 'venv'): - return get_render_func(venv.venv) - elif hasattr(venv, 'env'): - return get_render_func(venv.env) - - return None - - -def get_vec_normalize(venv): - if isinstance(venv, VecNormalize): - return venv - elif hasattr(venv, 'venv'): - return get_vec_normalize(venv.venv) - - return None - - -# Necessary for my KFAC implementation. -class AddBias(nn.Module): - def __init__(self, bias): - super(AddBias, self).__init__() - self._bias = nn.Parameter(bias.unsqueeze(1)) - - def forward(self, x): - if x.dim() == 2: - bias = self._bias.t().view(1, -1) - else: - bias = self._bias.t().view(1, -1, 1, 1) - - return x + bias - - -def update_linear_schedule(optimizer, epoch, total_num_epochs, initial_lr): - """Decreases the learning rate linearly""" - lr = initial_lr - (initial_lr * (epoch / float(total_num_epochs))) - for param_group in optimizer.param_groups: - param_group['lr'] = lr - - -def init(module, weight_init, bias_init, gain=1): - weight_init(module.weight.data, gain=gain) - bias_init(module.bias.data) - return module - - -def cleanup_log_dir(log_dir): - try: - os.makedirs(log_dir) - except OSError: - files = glob.glob(os.path.join(log_dir, '*.monitor.csv')) - for f in files: - os.remove(f) diff --git a/conspecfunction/requirements.txt b/conspecfunction/requirements.txt deleted file mode 100644 index 3fd7b3f..0000000 --- a/conspecfunction/requirements.txt +++ /dev/null @@ -1,121 +0,0 @@ -Package Version ----------------------- ----------- -absl-py 1.0.0 -argon2-cffi 21.3.0 -argon2-cffi-bindings 21.2.0 -astor 0.8.1 -attrs 21.4.0 -backcall 0.2.0 -beautifulsoup4 4.11.1 -bleach 5.0.0 -cached-property 1.5.2 -certifi 2021.10.8 -cffi 1.15.0 -charset-normalizer 2.0.12 -cloudpickle 2.0.0 -contextlib2 21.6.0 -cycler 0.11.0 -debugpy 1.6.0 -decorator 4.4.2 -defusedxml 0.7.1 -dm-sonnet 1.34 -dm-tree 0.1.7 -entrypoints 0.4 -fastjsonschema 2.15.3 -fonttools 4.33.3 -future 0.18.2 -gast 0.5.3 -google-pasta 0.2.0 -grpcio 1.46.0 -gym 0.16.0 -gym-notices 0.0.6 -h5py 3.6.0 -idna 3.3 -imageio 2.19.0 -imageio-ffmpeg 0.4.7 -importlib-metadata 4.11.3 -importlib-resources 5.7.1 -ipykernel 6.13.0 -ipython 7.33.0 -ipython-genutils 0.2.0 -ipywidgets 7.7.0 -jedi 0.18.1 -Jinja2 3.1.2 -jsonschema 4.5.1 -jupyter 1.0.0 -jupyter-client 7.3.0 -jupyter-console 6.4.3 -jupyter-core 4.10.0 -jupyterlab-pygments 0.2.2 -jupyterlab-widgets 1.1.0 -Keras-Applications 1.0.8 -Keras-Preprocessing 1.1.2 -kiwisolver 1.4.2 -Markdown 3.3.7 -MarkupSafe 2.1.1 -matplotlib 3.5.2 -matplotlib-inline 0.1.3 -mistune 0.8.4 -moviepy 1.0.3 -nbclient 0.6.2 -nbconvert 6.5.0 -nbformat 5.4.0 -nest-asyncio 1.5.5 -notebook 6.4.11 -numpy 1.21.5 -opt-einsum 3.3.0 -packaging 21.3 -pandas 1.1.5 -pandocfilters 1.5.0 -parso 0.8.3 -pexpect 4.8.0 -pickleshare 0.7.5 -Pillow 9.1.0 -pip 21.2.2 -proglog 0.1.10 -prometheus-client 0.14.1 -prompt-toolkit 3.0.29 -protobuf 3.20.1 -psutil 5.9.0 -ptyprocess 0.7.0 -pybullet 3.2.2 -pycolab 1.2 -pycparser 2.21 -pyglet 1.5.0 -Pygments 2.12.0 -pyparsing 3.0.8 -pyrsistent 0.18.1 -python-dateutil 2.8.2 -pytz 2022.1 -pyzmq 22.3.0 -qtconsole 5.3.0 -QtPy 2.1.0 -requests 2.27.1 -scipy 1.7.3 -semantic-version 2.9.0 -Send2Trash 1.8.0 -setuptools 61.2.0 -six 1.16.0 -soupsieve 2.3.2.post1 -stable-baselines3 0.9.0 -tensorboard 1.15.0 -tensorflow 1.15.0 -tensorflow-estimator 1.15.1 -tensorflow-probability 0.7.0 -termcolor 1.1.0 -terminado 0.13.3 -tinycss2 1.1.1 -torch 1.11.0 -tornado 6.1 -tqdm 4.64.0 -traitlets 5.1.1 -trfl 1.2.0 -typing_extensions 4.2.0 -urllib3 1.26.9 -wcwidth 0.2.5 -webencodings 0.5.1 -Werkzeug 2.1.2 -wheel 0.37.1 -widgetsnbextension 3.6.0 -wrapt 1.14.1 -zipp 3.8.0 diff --git a/conspecfunction/tvt/__pycache__/__init__.cpython-37.pyc b/conspecfunction/tvt/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index f8354a4..0000000 Binary files a/conspecfunction/tvt/__pycache__/__init__.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/__pycache__/batch_env.cpython-37.pyc b/conspecfunction/tvt/__pycache__/batch_env.cpython-37.pyc deleted file mode 100644 index c0b5a49..0000000 Binary files a/conspecfunction/tvt/__pycache__/batch_env.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/__pycache__/losses.cpython-37.pyc b/conspecfunction/tvt/__pycache__/losses.cpython-37.pyc deleted file mode 100644 index 6ece58a..0000000 Binary files a/conspecfunction/tvt/__pycache__/losses.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/__pycache__/memory.cpython-37.pyc b/conspecfunction/tvt/__pycache__/memory.cpython-37.pyc deleted file mode 100644 index ef8845a..0000000 Binary files a/conspecfunction/tvt/__pycache__/memory.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/__pycache__/nest_utils.cpython-37.pyc b/conspecfunction/tvt/__pycache__/nest_utils.cpython-37.pyc deleted file mode 100644 index 1b3e3ff..0000000 Binary files a/conspecfunction/tvt/__pycache__/nest_utils.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/__pycache__/rma_memobsendend.cpython-37.pyc b/conspecfunction/tvt/__pycache__/rma_memobsendend.cpython-37.pyc deleted file mode 100644 index f801ca2..0000000 Binary files a/conspecfunction/tvt/__pycache__/rma_memobsendend.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/__pycache__/tvt_rewards.cpython-37.pyc b/conspecfunction/tvt/__pycache__/tvt_rewards.cpython-37.pyc deleted file mode 100644 index cd40f36..0000000 Binary files a/conspecfunction/tvt/__pycache__/tvt_rewards.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/common.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/common.cpython-37.pyc deleted file mode 100644 index 5d9b5b2..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/common.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/env.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/env.cpython-37.pyc deleted file mode 100644 index 75b6afe..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/env.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/game.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/game.cpython-37.pyc deleted file mode 100644 index 8a9527e..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/game.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/key_to_door2.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/key_to_door2.cpython-37.pyc deleted file mode 100644 index 2a82e13..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/key_to_door2.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/key_to_door3.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/key_to_door3.cpython-37.pyc deleted file mode 100644 index f3f651a..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/key_to_door3.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/key_to_door4.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/key_to_door4.cpython-37.pyc deleted file mode 100644 index f31430b..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/key_to_door4.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_2keyseasy.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_2keyseasy.cpython-37.pyc deleted file mode 100644 index f170b7c..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_2keyseasy.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_3keyseasy.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_3keyseasy.cpython-37.pyc deleted file mode 100644 index 63d5fbb..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_3keyseasy.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_4keyseasy.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_4keyseasy.cpython-37.pyc deleted file mode 100644 index 1454261..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/key_to_door5_4keyseasy.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/key_to_doormany4.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/key_to_doormany4.cpython-37.pyc deleted file mode 100644 index 42e22d3..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/key_to_doormany4.cpython-37.pyc and /dev/null differ diff --git a/conspecfunction/tvt/pycolab/__pycache__/objects.cpython-37.pyc b/conspecfunction/tvt/pycolab/__pycache__/objects.cpython-37.pyc deleted file mode 100644 index 05110a9..0000000 Binary files a/conspecfunction/tvt/pycolab/__pycache__/objects.cpython-37.pyc and /dev/null differ diff --git a/data/max-cos-states-20240319-212006_key_to_door3_1_2500-epoch-70.png b/data/max-cos-states-20240319-212006_key_to_door3_1_2500-epoch-70.png new file mode 100644 index 0000000..af785de Binary files /dev/null and b/data/max-cos-states-20240319-212006_key_to_door3_1_2500-epoch-70.png differ diff --git a/data_collection.py b/data_collection.py new file mode 100644 index 0000000..41ffccb --- /dev/null +++ b/data_collection.py @@ -0,0 +1,121 @@ +import csv +import numpy as np +from Conspec.ConSpec import ConSpec +import torch +import matplotlib.pyplot as plt +import pdb + +def get_cosine_similarity(conspec, trajectories): + obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = trajectories + #obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch = trajectories + hidden = conspec.encoder.retrieve_hiddens(obs_batch, recurrent_hidden_states_batch,masks_batch, actions_batch) + #print('in cos scores in data', hidden.view(*obs_batchorig.size()[:2], -1).shape, obs_batchorig.shape) + default = (np.zeros((3,5,5)), -1) + state_prototypes = {0:default, 1:default, 2: default, 3: default, 4: default, 5:default, 6: default, 7:default} # prototype, states, cosine similarity + cos_max,_, cos_sim, _ = conspec.prototypes(hidden.view(*obs_batchorig.size()[:2], -1), -1) # [32,8] + + max_indices = torch.argmax(cos_max, axis=0) + + max_indices= max_indices.detach().cpu().numpy() # max_indices: torch.Size([8]): [26, 16, 1, 4, 5, 6, 7, 8] + pdb.set_trace() + + for proto, traj in enumerate(max_indices): + if cos_max[traj][proto] > state_prototypes[proto][1]: + print('replaced') + state_prototypes[proto] = (obs_batch[traj], cos_max[traj][proto]) + return cos_max, cos_sim, state_prototypes + +def read_data(file_path): + + # Read the data from a CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + data = list(reader) + + # Convert the data to a numpy array + data = np.array(data) + return data + +def collect_gfn_data(conspec, buffer, gfnpath, cospath): + # collect success and failure trajectories from the environment + # store their cosine similarity (per prototype) scores in a csv file + # each prototype appear as a variable in the csv file + # each row is a trajectory + data = [] + cos_data = [] + default = (np.zeros((3,5,5)), -1) + state_prototypes = {0:default, 1:default, 2: default, 3: default, 4: default, 5:default, 6: default, 7:default} # prototype, states, cosine similarity + + # Collect trajectories and compute their cosine similarities + for epoch in buffer: #only for when we use memories over epochs + cos_max, cos_scores, _ = get_cosine_similarity(conspec, epoch) # torch.Size([125, 32, 8]) + # cos_max = np.max(cos_sim, axis=0) # cos_max: torch.Size([32, 8]) + # max_indices = np.argmax(cos_max, axis=0) # max_indices: torch.Size([8]) + # epoch = epoch[0].detach().cpu().numpy() # states + # buffer = buffer[0].detach().cpu().numpy() + cos_scores = cos_scores.detach().cpu().numpy() + + # store the cosine similarity data for each trajectory + for i, t in enumerate(cos_scores): + for j, p in enumerate(t): + cos_sim_p = [1 if element > 0.6 else 0 for element in p] + if 1 in cos_sim_p: + data.append(cos_sim_p) + cos_data.append(p) + num_prototypes = cos_scores.shape[2] + + # Write the data to a CSV file + with open(gfnpath, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(['Prototype_' + str(i) for i in range(num_prototypes)]) + writer.writerows(data) + with open(cospath, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(cos_data) + # print(state_prototypes) + return state_prototypes + +def frozen_examplars(conspec, num_prototypes): + + for prototype in range(num_prototypes): + examplars = conspec.rollouts.retrieve_SFbuffer_frozen(prototype) # tuple of 5 + cos_sim = get_cosine_similarity(conspec, examplars).detach().cpu().numpy() # torch.Size([125, 32, 1]) + cos_max = np.max(cos_sim, axis=0) # cos_max: torch.Size([32, 8]) + max_indices = np.argmax(cos_max, axis=0) # max_indices: torch.Size([8]) + print('examplars shape', cos_sim.shape, cos_max) + + return examplars + +def visualize_states_prototypes(conspec, trajectories, env_name, seed, num_episodes): + '''does compare the observations to prototypes before visualizing''' + cos_sim = get_cosine_similarity(conspec, trajectories).detach().cpu().numpy() # torch.Size([125, 32, 8]) + cos_max = np.max(cos_sim, axis=0) # cos_max: torch.Size([32, 8]) + max_indices = np.argmax(cos_max, axis=0) # max_indices: torch.Size([8]) + + for prototype, traj_id in enumerate(max_indices): + print('traj_id, cos_max traj: ', traj_id, cos_max[traj_id]) + obs = trajectories[0][traj_id].detach().cpu().numpy() + obs = np.transpose(obs, (1, 2, 0)) + plt.imshow(obs.astype(np.uint8)) + plt.axis('off') # Remove the axis + plt.title("state corresponding to prototype " + str(prototype)) + plt.savefig("figures/{}_{}_seed_{}_eps_{}.png".format(env_name, prototype, seed, num_episodes)) + +def visualize(obs, env_name, seed, num_episodes, tag): + + for prototype, (state, cos) in obs.items(): + state = np.transpose(state, (1, 2, 0)) + plt.imshow(state.astype(np.uint8)) + plt.axis('off') # Remove the axis + plt.title("frozen state corres. to proto {} with cos sim: {}".format(prototype, cos)) + plt.savefig("figures/frozen/{}_frozen_examplars_{}_{}_seed_{}_eps_{}.png".format(tag, env_name, prototype, seed, num_episodes)) + +def visualize_per_proto(obs, prototype, env_name, seed, num_episodes, tag): + state = np.transpose(obs, (1, 2, 0)) + plt.imshow(state.astype(np.uint8)) + plt.axis('off') # Remove the axis + plt.title("frozen state corres. to proto {} with cos sim: {}".format(prototype)) + plt.savefig("figures/frozen/{}_examplars_{}_{}_seed_{}_eps_{}.png".format(tag, env_name, prototype, seed, num_episodes)) + + + diff --git a/conspecfunction/evaluation.py b/evaluation.py similarity index 100% rename from conspecfunction/evaluation.py rename to evaluation.py diff --git a/conspecfunction/generate_tmux_yaml.py b/generate_tmux_yaml.py similarity index 100% rename from conspecfunction/generate_tmux_yaml.py rename to generate_tmux_yaml.py diff --git a/main.py b/main.py new file mode 100644 index 0000000..832770c --- /dev/null +++ b/main.py @@ -0,0 +1,281 @@ +import os +from collections import deque +import torch +from misc import Logger, vis_prototypes +import matplotlib.pyplot as plt +import json +########packages to runthe pycolab game +from tvt import batch_env +from tvt import nest_utils +from tvt.pycolab import env as pycolab_env +from Conspec.ConSpec import ConSpec + +########packages to run the underlying RL algorithm, mostly identical to https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail +from a2c_ppo_acktr import algo, utils +from a2c_ppo_acktr.algo import gail +from a2c_ppo_acktr.arguments import get_args +from a2c_ppo_acktr.modelRL import Policy +from a2c_ppo_acktr.storage import RolloutStorage +from evaluation import evaluate +from tensorflow.contrib import framework as contrib_framework +import numpy as np +from six.moves import range +from six.moves import zip +import tensorflow.compat.v1 as tf +import datetime +import pdb +# from moviepy.editor import ImageSequenceClip + + +''' +The code below is nearly identical to https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail +With only minor additions of ConSpec-specific operations that have been commented ######## +''' + +def main(): + args = get_args() + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + + # Create the folder for results + timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + subfolder_name = f"{timestamp}_{args.pycolab_game}_{args.seed}_{args.num_episodes}_{args.num_prototypes}_{args.num_processes}" + base_directory = '/network/scratch/s/samieima/conspec_train' + full_path = os.path.join(base_directory, subfolder_name) + os.makedirs(full_path, exist_ok=True) + + + if args.cuda and torch.cuda.is_available() and args.cuda_deterministic: + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + + wandb_project_name = 'conspec' + #wandb.init(project=wandb_project_name) + proj_name = str(args.pycolab_game) +' seed'+ str(args.seed)+ ' episodes'+ str(args.num_episodes)+ ' prototype' + str(args.num_prototypes)+ ' num_processes' + str(args.num_processes) + logger = Logger( + exp_name=proj_name, + save_dir='/network/scratch/s/samieima/conspec_train', + print_every=1, + save_every=args.log_interval, + total_step=args.num_episodes, + print_to_stdout=True, + wandb_project_name=wandb_project_name, + wandb_tags=['multikeydoor'], + wandb_config=args, + ) + + log_dir = os.path.expanduser(args.log_dir) + eval_log_dir = log_dir + "_eval" + utils.cleanup_log_dir(log_dir) + utils.cleanup_log_dir(eval_log_dir) + + torch.set_num_threads(1) + device = torch.device("cuda:0" if args.cuda else "cpu") + + env_builder = pycolab_env.PycolabEnvironment + env_kwargs = { + 'game': args.pycolab_game, # key to door 4 + 'num_apples': args.pycolab_num_apples, # 10 + 'apple_reward': [args.pycolab_apple_reward_min, #0 + args.pycolab_apple_reward_max], #0 + 'fix_apple_reward_in_episode': args.pycolab_fix_apple_reward_in_episode, + 'final_reward': args.pycolab_final_reward, + 'crop': args.pycolab_crop + } + env = batch_env.BatchEnv(args.num_processes, env_builder, **env_kwargs) + + ep_length = env.episode_length + args.num_steps = ep_length + envs = env + obsspace = (3,5,5) #env.observation_shape + actor_critic = Policy( + obsspace, + env.num_actions, + base_kwargs={'recurrent': args.recurrent_policy}) # envs.observation_space.shape, + actor_critic.to(device) + + ################################### + ##decide on which underlying RL agent to use - ppo or a2c. But any other RL agent of choice should also work + if args.algo == 'a2c': + agent = algo.A2C_ACKTR( + actor_critic, + args.value_loss_coef, + args.entropy_coef, + lr=args.lr, + eps=args.eps, + alpha=args.alpha, + max_grad_norm=args.max_grad_norm) + + elif args.algo == 'ppo': + agent = algo.PPO( + actor_critic, + args.clip_param, + args.ppo_epoch, + args.num_mini_batch, + args.value_loss_coef, + args.entropy_coef, + lr=args.lr, + eps=args.eps, + max_grad_norm=args.max_grad_norm) + + ###############CONSPEC FUNCTIONS############################## + ''' + Here, the main ConSpec class is loaded. All the relevant ConSpec functions and objects are contained in this class. + ''' + conspecfunction = ConSpec(args, obsspace, env.num_actions, device) + # file_path='datasets/frozen/trajectory_data_gfn_{}_seed_{}_{}_episodes_frozen.csv'.format(args.pycolab_game, args.seed, args.num_episodes) + # cos_file_path='datasets/frozen/cos_sim_{}_seed_{}_{}_episodes_frozen.csv'.format(args.pycolab_game, args.seed ,args.num_episodes) + + ############################################################## + print('steps', args.num_steps) + rollouts = RolloutStorage(args.num_steps, args.num_processes, + obsspace, env.num_actions, + actor_critic.recurrent_hidden_state_size, args.num_prototypes) # envs.observation_space.shape + rollouts.to(device) + + obs, _ = envs.reset() + obs = (torch.from_numpy(obs)).permute((0, 3, 1, 2)).to(device) + rollouts.obs[0].copy_(obs) + rollouts.to(device) + + episode_rewards = deque(maxlen=10) + num_updates = int(args.num_env_steps) // args.num_steps // args.num_processes + logger.start() + env_frames = {i: [] for i in range(args.num_processes)} # to make a video of training + + for episode in range(args.num_episodes): + logger.step() + obs, _ = envs.reset() + obs = (torch.from_numpy(obs)).permute((0, 3, 1, 2)).to(device) #torch.Size([num_processes, 3, 5, 5]) + rollouts.obs[0].copy_(obs) + donetotal = np.zeros((args.num_processes,)) # .to(device) + if args.use_linear_lr_decay: + # decrease learning rate linearly + utils.update_linear_schedule( + agent.optimizer, episode, num_updates, + agent.optimizer.lr if args.algo == "acktr" else args.lr) + + for step in range(args.num_steps): + # Sample actions + with torch.no_grad(): + value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( + rollouts.obs[step], rollouts.recurrent_hidden_states[step], + rollouts.masks[step]) + + obs, reward = envs.step(action) + obs = torch.from_numpy(obs).permute((0, 3, 1, 2)).to(device) + reward = torch.from_numpy(reward).reshape(-1, 1) + done = donetotal + masks = torch.FloatTensor( + [[0.0] if done_ else [1.0] for done_ in done]) + bad_masks = masks + rollouts.insert(obs, recurrent_hidden_states, action, + action_log_prob, value, reward, masks, bad_masks) + + for i in range(args.num_processes): + vobs = obs[i].cpu().detach().numpy() + vobs = vobs.transpose(1, 2, 0) + env_frames[i].append(vobs) + + + with torch.no_grad(): + next_value = actor_critic.get_value( + rollouts.obs[-1], rollouts.recurrent_hidden_states[-1], + rollouts.masks[-1]).detach() + # now compute new rewards + rewardstotal = rollouts.retrieveR() + episode_rewards.append(rewardstotal.sum(0).mean().cpu().detach().numpy()) + + ###############CONSPEC FUNCTIONS############################## + ''' + The purpose here is to: + 1. retrieve the current minibatch of trajectory (including its observations, rewards, hidden states, actions, masks) + 2. "do everything" that ConSpec needs to do internally for training, and output the intrinsic + extrinsic reward for the current minibatch of trajectories + 3. store this total reward in the memory buffer + ''' + + obstotal, rewardtotal, recurrent_hidden_statestotal, actiontotal, maskstotal = rollouts.release() + reward_intrinsic_extrinsic = conspecfunction.do_everything(obstotal, recurrent_hidden_statestotal, actiontotal, rewardtotal, maskstotal) + + rollouts.storereward(reward_intrinsic_extrinsic) + ############################################################## + + rollouts.compute_returns(next_value, args.use_gae, args.gamma, + args.gae_lambda, args.use_proper_time_limits) + + print('epoch', episode) + value_loss, action_loss, dist_entropy= agent.update(rollouts) + rollouts.after_update() + + + if episode % args.log_interval == 0 and len(episode_rewards) > 1: + logger.meter("results", "R", rewardstotal[-10:,:].sum(0).mean().cpu().detach().numpy()) + logger.meter("results", "dist_entropy", dist_entropy) + logger.meter("results", "value_loss", value_loss) + logger.meter("results", "action_loss", action_loss) + + if episode > args.start_checkpoint and (episode) % args.checkpoint_interval == 0: + buffer = { + 'obs': rollouts.obs, + 'rewards': rollouts.rewards, + 'hidden_states': rollouts.recurrent_hidden_states, + 'actions': rollouts.actions, + 'masks': rollouts.masks, + 'bad_masks': rollouts.bad_masks, + 'value_preds': rollouts.value_preds, + } + sf_buffer = conspecfunction.rollouts.retrieve_SFbuffer() + conspec_rollouts = { + 'obs': sf_buffer[0], + 'rewards': sf_buffer[5], + 'hidden_states': sf_buffer[1], + 'actions': sf_buffer[3], + 'masks': sf_buffer[2], + 'bad_masks': sf_buffer[2], + 'value_preds': sf_buffer[4], + } + tensor_proto_list = [p.data for p in conspecfunction.prototypes.prototypes] + model_checkpoint = { + 'epoch': episode, + 'encoder_state_dict': conspecfunction.encoder.state_dict(), + 'actor_critic_state_dict': actor_critic.state_dict(), + 'optimizer_conspec_state_dict': conspecfunction.optimizerConSpec.state_dict(), + 'optimizer_ppo_state_dict': agent.optimizer.state_dict(), + 'prototypes_state_dict': tensor_proto_list, + 'prototypes': conspecfunction.prototypes.prototypes.state_dict(), + } + cos_checkpoint = { + 'cos_max_scores' : conspecfunction.rollouts.cos_max_scores, + 'max_indices' : conspecfunction.rollouts.max_indx, + 'cos_scores' : conspecfunction.rollouts.cos_scores, + # 'cos_success' : conspecfunction.rollouts.cos_score_pos, + # 'cos_failure' : conspecfunction.rollouts.cos_score_neg, + } + + print('saving checkpoints....') + checkpoint_path = os.path.join(full_path, f'model_checkpoint_epoch_{episode}.pth') + buffer_path = os.path.join(full_path, f'buffer_epoch_{episode}.pth') + conspec_rollouts_path = os.path.join(full_path, f'conspec_rollouts_epoch_{episode}.pth') + cos_path = os.path.join(full_path, f'cos_sim_epoch_{episode}.pth') + + + torch.save(model_checkpoint, checkpoint_path) + print('model checkpoint saved') + + torch.save(buffer, buffer_path) + print('buffer saved') + + torch.save(conspec_rollouts, conspec_rollouts_path) + print('success/failure buffers saved') + + torch.save(cos_checkpoint, cos_path) + print('cosine similarity saved') + + frams_dict_path = os.path.join(full_path, f'env_frames.npz') + np.savez(frams_dict_path, **env_frames) + + logger.finish() + +if __name__ == "__main__": + main() diff --git a/conspecfunction/main.py b/main_eval.py similarity index 63% rename from conspecfunction/main.py rename to main_eval.py index b300124..7baf984 100644 --- a/conspecfunction/main.py +++ b/main_eval.py @@ -1,202 +1,264 @@ -import copy -import glob -import os -import time -from collections import deque -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -import sys -import numpy -import wandb -from misc import Logger, makedir - -########packages to runthe pycolab game -from tvt import batch_env -from tvt import nest_utils -from tvt.pycolab import env as pycolab_env - -########packages to run the underlying RL algorithm, mostly identical to https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail -from a2c_ppo_acktr import algo, utils -from a2c_ppo_acktr.algo import gail -from a2c_ppo_acktr.arguments import get_args -from a2c_ppo_acktr.modelRL import Policy -from a2c_ppo_acktr.storage import RolloutStorage -from evaluation import evaluate -from tensorflow.contrib import framework as contrib_framework -import numpy as np -from six.moves import range -from six.moves import zip -import tensorflow.compat.v1 as tf - - - -''' -The code below is nearly identical to https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail -With only minor additions of ConSpec-specific operations that have been commented ######## -''' - -def main(): - args = get_args() - torch.manual_seed(args.seed) - torch.cuda.manual_seed_all(args.seed) - - if args.cuda and torch.cuda.is_available() and args.cuda_deterministic: - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - - wandb_project_name = 'conspec' - wandb.init(name='sunchipsster', project=wandb_project_name) - proj_name = 'Results' + str(args.pycolab_game) + str(args.seed) - logger = Logger( - exp_name=proj_name, - save_dir='/home/mila/c/chen.sun/scratch/conspec', - print_every=1, - save_every=10000, - total_step=10000, - print_to_stdout=True, - wandb_project_name=wandb_project_name, - wandb_tags=[], - wandb_config=args, - ) - - log_dir = os.path.expanduser(args.log_dir) - eval_log_dir = log_dir + "_eval" - utils.cleanup_log_dir(log_dir) - utils.cleanup_log_dir(eval_log_dir) - - torch.set_num_threads(1) - device = torch.device("cuda:0" if args.cuda else "cpu") - - env_builder = pycolab_env.PycolabEnvironment - env_kwargs = { - 'game': args.pycolab_game, - 'num_apples': args.pycolab_num_apples, - 'apple_reward': [args.pycolab_apple_reward_min, - args.pycolab_apple_reward_max], - 'fix_apple_reward_in_episode': args.pycolab_fix_apple_reward_in_episode, - 'final_reward': args.pycolab_final_reward, - 'crop': args.pycolab_crop - } - env = batch_env.BatchEnv(args.num_processes, env_builder, **env_kwargs) - ep_length = env.episode_length - args.num_steps = ep_length - - envs = env - obsspace = (3,5,5) #env.observation_shape - actor_critic = Policy( - obsspace, - env.num_actions, - base_kwargs={'recurrent': args.recurrent_policy}) # envs.observation_space.shape, - actor_critic.to(device) - - ################################### - ##decide on which underlying RL agent to use - ppo or a2c. But any other RL agent of choice should also work - if args.algo == 'a2c': - agent = algo.A2C_ACKTR( - actor_critic, - args.value_loss_coef, - args.entropy_coef, - lr=args.lr, - eps=args.eps, - alpha=args.alpha, - max_grad_norm=args.max_grad_norm) - - elif args.algo == 'ppo': - agent = algo.PPO( - actor_critic, - args.clip_param, - args.ppo_epoch, - args.num_mini_batch, - args.value_loss_coef, - args.entropy_coef, - lr=args.lr, - eps=args.eps, - max_grad_norm=args.max_grad_norm) - - ###############CONSPEC FUNCTIONS############################## - ''' - Here, the main ConSpec class is loaded. All the relevant ConSpec functions and objects are contained in this class. - ''' - from Conspec.ConSpec import ConSpec - conspecfunction = ConSpec(args, obsspace, env.num_actions, device) - ############################################################## - - rollouts = RolloutStorage(args.num_steps, args.num_processes, - obsspace, env.num_actions, - actor_critic.recurrent_hidden_state_size, args.num_prototypes) # envs.observation_space.shape - rollouts.to(device) - - obs, _ = envs.reset() - obs = (torch.from_numpy(obs)).permute((0, 3, 1, 2)).to(device) - rollouts.obs[0].copy_(obs) - rollouts.to(device) - - episode_rewards = deque(maxlen=10) - num_updates = int(args.num_env_steps) // args.num_steps // args.num_processes - - logger.start() - for j in range(10000): - logger.step() - obs, _ = envs.reset() - obs = (torch.from_numpy(obs)).permute((0, 3, 1, 2)).to(device) - rollouts.obs[0].copy_(obs) - donetotal = np.zeros((args.num_processes,)) # .to(device) - if args.use_linear_lr_decay: - # decrease learning rate linearly - utils.update_linear_schedule( - agent.optimizer, j, num_updates, - agent.optimizer.lr if args.algo == "acktr" else args.lr) - - for step in range(args.num_steps): - # Sample actions - with torch.no_grad(): - value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( - rollouts.obs[step], rollouts.recurrent_hidden_states[step], - rollouts.masks[step]) - - obs, reward = envs.step(action) - obs = torch.from_numpy(obs).permute((0, 3, 1, 2)).to(device) - reward = torch.from_numpy(reward).reshape(-1, 1) - done = donetotal - masks = torch.FloatTensor( - [[0.0] if done_ else [1.0] for done_ in done]) - bad_masks = masks - rollouts.insert(obs, recurrent_hidden_states, action, - action_log_prob, value, reward, masks, bad_masks) - - with torch.no_grad(): - next_value = actor_critic.get_value( - rollouts.obs[-1], rollouts.recurrent_hidden_states[-1], - rollouts.masks[-1]).detach() - # now compute new rewards - rewardstotal = rollouts.retrieveR() - episode_rewards.append(rewardstotal.sum(0).mean().cpu().detach().numpy()) - - ###############CONSPEC FUNCTIONS############################## - ''' - the purpose here is to: - 1. retrieve the current minibatch of trajectory (including its observations, rewards, hidden states, actions, masks) - 2. "do everything" that ConSpec needs to do internally for training, and output the intrinsic + extrinsic reward for the current minibatch of trajectories - 3. store this total reward in the memory buffer - ''' - obstotal, rewardtotal, recurrent_hidden_statestotal, actiontotal, maskstotal = rollouts.release() - reward_intrinsic_extrinsic = conspecfunction.do_everything(obstotal, recurrent_hidden_statestotal, actiontotal, rewardtotal, maskstotal) - rollouts.storereward(reward_intrinsic_extrinsic) - ############################################################## - - rollouts.compute_returns(next_value, args.use_gae, args.gamma, - args.gae_lambda, args.use_proper_time_limits) - - value_loss, action_loss, dist_entropy= agent.update(rollouts) - rollouts.after_update() - - if j % args.log_interval == 0 and len(episode_rewards) > 1: - logger.meter("results", "R", rewardstotal[-10:,:].sum(0).mean().cpu().detach().numpy()) - logger.meter("results", "dist_entropy", dist_entropy) - logger.meter("results", "value_loss", value_loss) - logger.meter("results", "action_loss", action_loss) - logger.finish() -if __name__ == "__main__": - main() +import os +from collections import deque +import torch +from misc import Logger, vis_prototypes +import matplotlib.pyplot as plt +import json +########packages to runthe pycolab game +from tvt import batch_env +from tvt import nest_utils +from tvt.pycolab import env as pycolab_env +from Conspec.ConSpec import ConSpec + +########packages to run the underlying RL algorithm, mostly identical to https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail +from a2c_ppo_acktr import algo, utils +from a2c_ppo_acktr.algo import gail +from a2c_ppo_acktr.arguments import get_args +from a2c_ppo_acktr.modelRL import Policy +from a2c_ppo_acktr.storage import RolloutStorage +from evaluation import evaluate +from tensorflow.contrib import framework as contrib_framework +import numpy as np +from six.moves import range +from six.moves import zip +import tensorflow.compat.v1 as tf +# from data_collection import collect_gfn_data, read_data, get_cosine_similarity, visualize +import pickle +import datetime +from pathlib import Path + + + +def main_eval(): + args = get_args() + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + + # Create the folder for results + timestamp = datetime.datetime.now().strftime("%Y%m%d") + subfolder_name = f"{timestamp}_{args.pycolab_game}_{args.seed}_{args.num_episodes}" + base_directory = '/network/scratch/s/samieima/conspec_eval' + full_path = os.path.join(base_directory, subfolder_name) + os.makedirs(full_path, exist_ok=True) + + + if args.cuda and torch.cuda.is_available() and args.cuda_deterministic: + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + + wandb_project_name = 'conspec-eval' + #wandb.init(project=wandb_project_name) + proj_name = str(args.pycolab_game) +' seed'+ str(args.seed)+ ' episodes'+ str(args.num_episodes) + logger = Logger( + exp_name=proj_name, + save_dir='/network/scratch/s/samieima/conspec_eval', + print_every=1, + save_every=args.log_interval, + total_step=args.num_episodes, + print_to_stdout=True, + wandb_project_name=wandb_project_name, + wandb_tags=['multikeydoor'], + wandb_config=args, + ) + + log_dir = os.path.expanduser(args.log_dir) + eval_log_dir = log_dir + "_eval" + utils.cleanup_log_dir(log_dir) + utils.cleanup_log_dir(eval_log_dir) + + torch.set_num_threads(1) + device = torch.device("cuda:0" if args.cuda else "cpu") + + env_builder = pycolab_env.PycolabEnvironment + env_kwargs = { + 'game': args.pycolab_game, # key to door 4 + 'num_apples': args.pycolab_num_apples, # 10 + 'apple_reward': [args.pycolab_apple_reward_min, #0 + args.pycolab_apple_reward_max], #0 + 'fix_apple_reward_in_episode': args.pycolab_fix_apple_reward_in_episode, + 'final_reward': args.pycolab_final_reward, + 'crop': args.pycolab_crop + } + env = batch_env.BatchEnv(args.num_processes, env_builder, **env_kwargs) + + ep_length = env.episode_length + args.num_steps = ep_length + envs = env + obsspace = (3,5,5) #env.observation_shape + actor_critic = Policy( + obsspace, + env.num_actions, + base_kwargs={'recurrent': args.recurrent_policy}) # envs.observation_space.shape, + actor_critic.to(device) + + ################################### + ##decide on which underlying RL agent to use - ppo or a2c. But any other RL agent of choice should also work + if args.algo == 'a2c': + agent = algo.A2C_ACKTR( + actor_critic, + args.value_loss_coef, + args.entropy_coef, + lr=args.lr, + eps=args.eps, + alpha=args.alpha, + max_grad_norm=args.max_grad_norm) + + elif args.algo == 'ppo': + agent = algo.PPO( + actor_critic, + args.clip_param, + args.ppo_epoch, + args.num_mini_batch, + args.value_loss_coef, + args.entropy_coef, + lr=args.lr, + eps=args.eps, + max_grad_norm=args.max_grad_norm) + + + ###############CONSPEC FUNCTIONS############################## + ''' + Here, the main ConSpec class is loaded. All the relevant ConSpec functions and objects are contained in this class. + ''' + conspecfunction = ConSpec(args, obsspace, env.num_actions, device) + + ############################################################## + print('steps', args.num_steps) + rollouts = RolloutStorage(args.num_steps, args.num_processes, + obsspace, env.num_actions, + actor_critic.recurrent_hidden_state_size, args.num_prototypes) # envs.observation_space.shape + rollouts.to(device) + + obs, _ = envs.reset() + obs = (torch.from_numpy(obs)).permute((0, 3, 1, 2)).to(device) + rollouts.obs[0].copy_(obs) + rollouts.to(device) + + episode_rewards = deque(maxlen=10) + num_updates = int(args.num_env_steps) // args.num_steps // args.num_processes + logger.start() + + checkpoint = torch.load('data/kd3/checkpoint_epoch_1999.pth') + actor_critic.load_state_dict(checkpoint['actor_critic_state_dict']) + agent.optimizer.load_state_dict(checkpoint['optimizer_ppo_state_dict']) + conspecfunction.optimizerConSpec.load_state_dict(checkpoint['optimizer_conspec_state_dict']) + proto_list = checkpoint['prototypes_state_dict'] + for idx, proto in enumerate(conspecfunction.prototypes.prototypes): + proto.data = proto_list[idx] + + + for episode in range(args.num_episodes): + logger.step() + obs, _ = envs.reset() + obs = (torch.from_numpy(obs)).permute((0, 3, 1, 2)).to(device) + rollouts.obs[0].copy_(obs) + donetotal = np.zeros((args.num_processes,)) # .to(device) + if args.use_linear_lr_decay: + # decrease learning rate linearly + utils.update_linear_schedule( + agent.optimizer, episode, num_updates, + agent.optimizer.lr if args.algo == "acktr" else args.lr) + + for step in range(args.num_steps): + # Sample actions + with torch.no_grad(): + value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( + rollouts.obs[step], rollouts.recurrent_hidden_states[step], + rollouts.masks[step]) + + obs, reward = envs.step(action) + obs = torch.from_numpy(obs).permute((0, 3, 1, 2)).to(device) + reward = torch.from_numpy(reward).reshape(-1, 1) + done = donetotal + masks = torch.FloatTensor( + [[0.0] if done_ else [1.0] for done_ in done]) + bad_masks = masks + rollouts.insert(obs, recurrent_hidden_states, action, + action_log_prob, value, reward, masks, bad_masks) + + with torch.no_grad(): + next_value = actor_critic.get_value( + rollouts.obs[-1], rollouts.recurrent_hidden_states[-1], + rollouts.masks[-1]).detach() + # now compute new rewards + rewardstotal = rollouts.retrieveR() + episode_rewards.append(rewardstotal.sum(0).mean().cpu().detach().numpy()) + + ###############CONSPEC FUNCTIONS############################## + ''' + The purpose here is to: + 1. retrieve the current minibatch of trajectory (including its observations, rewards, hidden states, actions, masks) + 2. "do everything" that ConSpec needs to do internally for training, and output the intrinsic + extrinsic reward for the current minibatch of trajectories + 3. store this total reward in the memory buffer + ''' + + obstotal, rewardtotal, recurrent_hidden_statestotal, actiontotal, maskstotal = rollouts.release() + reward_intrinsic_extrinsic = conspecfunction.do_everything(obstotal, recurrent_hidden_statestotal, actiontotal, rewardtotal, maskstotal) + + rollouts.storereward(reward_intrinsic_extrinsic) + ############################################################## + + rollouts.compute_returns(next_value, args.use_gae, args.gamma, + args.gae_lambda, args.use_proper_time_limits) + + value_loss, action_loss, dist_entropy= agent.update(rollouts) + rollouts.after_update() + + + if episode % args.log_interval == 0 and len(episode_rewards) > 0: + logger.meter("results", "R", rewardstotal[-10:,:].sum(0).mean().cpu().detach().numpy()) + logger.meter("results", "dist_entropy", dist_entropy) + logger.meter("results", "value_loss", value_loss) + logger.meter("results", "action_loss", action_loss) + + if episode > args.start_checkpoint and (episode) % args.checkpoint_interval == 0: + buffer = { + 'obs': rollouts.obs, + 'rewards': rollouts.rewards, + 'hidden_states': rollouts.recurrent_hidden_states, + 'actions': rollouts.actions, + 'masks': rollouts.masks, + 'bad_masks': rollouts.bad_masks, + 'value_preds': rollouts.value_preds, + } + sf_buffer = conspecfunction.rollouts.retrieve_SFbuffer() + conspec_rollouts = { + 'obs': sf_buffer[0], + 'rewards': sf_buffer[5], + 'hidden_states': sf_buffer[1], + 'actions': sf_buffer[3], + 'masks': sf_buffer[2], + 'bad_masks': sf_buffer[2], + 'value_preds': sf_buffer[4], + } + cos_checkpoint = { + 'cos_max_scores' : conspecfunction.rollouts.cos_max_scores, + 'max_indices' : conspecfunction.rollouts.max_indx, + 'cos_scores' : conspecfunction.rollouts.cos_scores, + # 'cos_success' : conspecfunction.rollouts.cos_score_pos, + # 'cos_failure' : conspecfunction.rollouts.cos_score_neg, + } + + print('saving checkpoints....') + buffer_path = os.path.join(full_path, f'buffer_epoch_{episode}.pth') + conspec_rollouts_path = os.path.join(full_path, f'conspec_rollouts_epoch_{episode}.pth') + cos_path = os.path.join(full_path, f'cos_sim_epoch_{episode}.pth') + + torch.save(buffer, buffer_path) + print('buffer saved') + + torch.save(conspec_rollouts, conspec_rollouts_path) + print('success/failure buffers saved') + + torch.save(cos_checkpoint, cos_path) + print('cosine similarity saved') + + + # pickle data files + + + logger.finish() + +if __name__ == "__main__": + main_eval() diff --git a/conspecfunction/misc.py b/misc.py similarity index 72% rename from conspecfunction/misc.py rename to misc.py index 20a73b4..fcc20c8 100644 --- a/conspecfunction/misc.py +++ b/misc.py @@ -5,6 +5,8 @@ from datetime import datetime import torch.nn as nn import numpy as np +import matplotlib.pyplot as plt +from moviepy.editor import ImageSequenceClip def makedir(path): if not os.path.exists(path): @@ -13,6 +15,67 @@ def makedir(path): else: print(path, "already exist!") +def serialize_object(obj): + return {attr: getattr(obj, attr) for attr in dir(obj) if not attr.startswith('__') and not callable(getattr(obj, attr))} + +def visuali_obs(batch_env): + # Assuming batch_env is your BatchEnv instance + # and it has a method get_observations() that returns a list of observations + observations = batch_env.reset() + + # Number of environments + num_envs = len(observations) + + # Create subplots + fig, axes = plt.subplots(1, num_envs, figsize=(num_envs * 5, 5)) + + for i, obs in enumerate(observations): + ax = axes[i] + ax.imshow(obs) # Replace this with appropriate plotting code for your observations + ax.set_title(f"Environment {i+1}") + plt.savefig('observation.png') + wandb.log({"observation Plot": wandb.Image('observation.png')}) + +def vis_prototypes(conspec, args): + import time + current_time = time.strftime("%H:%M:%S") + prototypes_used, frequencies = conspec.rollouts.retrieve_prototypes_used() + print('prototypes_used', prototypes_used) + print('frequencies', frequencies) + + # obs = obs_batch[:, :, :3, :, :] + # o1, o2, o3, o4, o5 = obs.shape + + num_success_samples = conspec.rollouts.success_sample + ##obs = [600, 16, 1, 54, 64] + + for i in range(conspec.num_prototypes): # indtop5.size()[0]): # top5 + obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, reward_batch = conspec.rollouts.retrieve_SFbuffer_frozen(i) + cos_sim_total_max, cost_prototype, cos_scores, ـ = conspec.calc_cos_scores(obs_batch, recurrent_hidden_states_batch, masks_batch, actions_batch, obs_batchorig, i) + cos_max, indmaxes0 = torch.max(cos_scores, dim=0) # cos_max: torch.Size([16, 8]) + # num_traj = obs_batch.shape[0] + # success_traj = int(num_traj/2) + + obs = obs_batch.view(args.num_steps, args.num_processes, *obs_batch.shape[1:]) + # print('cos_max', cos_max.shape) # torch.Size([32, 8]) + # print('indmaxes0', indmaxes0.shape) # torch.Size([32, 8]) + # indmaxes0 = torch.argmax(cos_sim_total_max, dim=0) + if prototypes_used[i] == 0: + indmaxestop5 = indmaxes0 # indtop5[jjjj, :, :].squeeze() + o1, o2, o3, o4, o5 = obs.shape + # for iii in range(self.head): + obsgathered0 = torch.gather(obs, 0, indmaxestop5[:, i].reshape(1, -1, 1, 1, 1).repeat(o1, 1, o3, o4, o5))[0] + print('obsgather', obsgathered0.shape) + imgs = (np.asarray(obsgathered0.cpu().detach().numpy()))[:, :, :, :].transpose((0, 2, 3, 1)) # .repeat(3,axis=-1) # .repeat(3,axis=-1) + fig = plt.figure(figsize=(20, 20)) # first coor = LENGTH + for j in range(num_success_samples*2): # 16 [64,16] + fig.add_subplot(4, 8, j + 1) + plt.imshow(imgs[j, :, :, :].squeeze() / 255., interpolation='nearest') # , cmap='gray') # (81, 48, 5, 5, 3) + plt.xlabel('R_' + str(reward_batch.sum(0)[j].cpu().detach().numpy()) + ' c_' + str( + cos_sim_total_max[j, i].cpu().detach().numpy()) + 'p_' + str(indmaxes0[j, i].cpu().detach().numpy())) + plt.savefig('moviemaxfirst_visualizeTVTX' + "S" + str(current_time) + str(args.pycolab_game) + 'CLparam' + str(args.intrinsicR_scale) + 'lrConSpec' + str(args.lrConSpec) + 'seed' + str(args.seed) + 'topk' + str(j) + 'prototype' + str(i) + 'top5costX.pdf') + plt.close() + class Logger: def __init__( self, @@ -46,7 +109,7 @@ def __init__( self.objects_to_save = dict() if "/" in exp_suffix: exp_suffix = "_".join(exp_suffix.split("/")[:-1]) - wandb.init(entity="sunchipsster", project=wandb_project_name, name=exp_name + "_" + exp_suffix, tags=wandb_tags, reinit=True) + wandb.init(entity="mandanasmi", project=wandb_project_name, name=exp_name + "_" + exp_suffix, tags=wandb_tags, reinit=True) wandb.config.update(wandb_config) def register_model_to_save(self, model, name): diff --git a/conspecfunction/misc_util.py b/misc_util.py similarity index 88% rename from conspecfunction/misc_util.py rename to misc_util.py index c0a4238..ab3c7d6 100644 --- a/conspecfunction/misc_util.py +++ b/misc_util.py @@ -4,6 +4,8 @@ import torch import torch.nn as nn +def serialize_object(obj): + return {attr: getattr(obj, attr) for attr in dir(obj) if not attr.startswith('__') and not callable(getattr(obj, attr))} def set_global_seeds(seed): torch.manual_seed(seed) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..48bbc24 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +# Core dependencies for ConSpec +torch>=1.10.0 +torchvision>=0.11.0 +numpy>=1.21.0 +matplotlib>=3.5.0 +tqdm>=4.64.0 + +# RL and environment dependencies +gym>=0.16.0 +pycolab>=1.2 +stable-baselines3>=0.9.0 + +# DeepMind dependencies for pycolab environments +dm-sonnet>=1.34 +dm-tree>=0.1.7 +trfl>=1.2.0 + +# TensorFlow (for some utilities) +tensorflow>=1.15.0 +tensorboard>=1.15.0 + +# Image processing +Pillow>=9.1.0 +imageio>=2.19.0 +moviepy>=1.0.3 + +# Utilities +scipy>=1.7.0 +pandas>=1.1.0 +psutil>=5.9.0 +protobuf>=3.20.0 \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..28458cd --- /dev/null +++ b/run.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +#SBATCH --partition=long +#SBATCH --gres=gpu:rtx8000:1 +#SBATCH --mem=16GB +#SBATCH --time=24:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --output=sbatch_out/conspec_key_to_door3_train.%A.%a.out +#SBATCH --error=sbatch_err/conspec_key_to_door3_train.%A.%a.err +#SBATCH --job-name=conspec_key_to_door3_train + +echo "Date: $(date)" + + +module load anaconda/3 +module load cuda/10.0/cudnn/7.6 +conda activate py37tf15 + +# Stage dataset into $SLURM_TMPDIR + +python -u main.py --pycolab_game key_to_door4 --num_episodes 10000 --seed 1 --num-processes 32 --num_prototypes 6 --checkpoint_interval 1 --start_checkpoint 0 + + diff --git a/run_eval.sh b/run_eval.sh new file mode 100755 index 0000000..f2ca359 --- /dev/null +++ b/run_eval.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +#SBATCH --partition=long +#SBATCH --gres=gpu:rtx8000:1 +#SBATCH --mem=16GB +#SBATCH --time=48:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --output=sbatch_out/conspec_key_to_door2_eval.%A.%a.out +#SBATCH --error=sbatch_err/conspec_key_to_door2_eval.%A.%a.err +#SBATCH --job-name=conspec_key_to_door2_eval + +echo "Date: $(date)" + + +module load anaconda/3 +module load cuda/10.0/cudnn/7.6 +conda activate py37tf15 + +# Stage dataset into $SLURM_TMPDIR + +python -u main_eval.py --pycolab_game key_to_door2 --num_episodes 5000 --seed 1 --num-processes 32 --checkpoint_interval 1 --start_checkpoint 0 + + diff --git a/conspecfunction/tvt/__init__.py b/tvt/__init__.py similarity index 100% rename from conspecfunction/tvt/__init__.py rename to tvt/__init__.py diff --git a/conspecfunction/tvt/batch_env.py b/tvt/batch_env.py similarity index 100% rename from conspecfunction/tvt/batch_env.py rename to tvt/batch_env.py diff --git a/conspecfunction/tvt/dmlab/README.md b/tvt/dmlab/README.md similarity index 100% rename from conspecfunction/tvt/dmlab/README.md rename to tvt/dmlab/README.md diff --git a/conspecfunction/tvt/dmlab/active_visual_match.lua b/tvt/dmlab/active_visual_match.lua similarity index 100% rename from conspecfunction/tvt/dmlab/active_visual_match.lua rename to tvt/dmlab/active_visual_match.lua diff --git a/conspecfunction/tvt/dmlab/image_utils.lua b/tvt/dmlab/image_utils.lua similarity index 100% rename from conspecfunction/tvt/dmlab/image_utils.lua rename to tvt/dmlab/image_utils.lua diff --git a/conspecfunction/tvt/dmlab/key_to_door.lua b/tvt/dmlab/key_to_door.lua similarity index 100% rename from conspecfunction/tvt/dmlab/key_to_door.lua rename to tvt/dmlab/key_to_door.lua diff --git a/conspecfunction/tvt/dmlab/key_to_door_bluekey.lua b/tvt/dmlab/key_to_door_bluekey.lua similarity index 100% rename from conspecfunction/tvt/dmlab/key_to_door_bluekey.lua rename to tvt/dmlab/key_to_door_bluekey.lua diff --git a/conspecfunction/tvt/dmlab/key_to_door_factory.lua b/tvt/dmlab/key_to_door_factory.lua similarity index 100% rename from conspecfunction/tvt/dmlab/key_to_door_factory.lua rename to tvt/dmlab/key_to_door_factory.lua diff --git a/conspecfunction/tvt/dmlab/key_to_door_to_match.lua b/tvt/dmlab/key_to_door_to_match.lua similarity index 100% rename from conspecfunction/tvt/dmlab/key_to_door_to_match.lua rename to tvt/dmlab/key_to_door_to_match.lua diff --git a/conspecfunction/tvt/dmlab/latent_information_acquisition.lua b/tvt/dmlab/latent_information_acquisition.lua similarity index 100% rename from conspecfunction/tvt/dmlab/latent_information_acquisition.lua rename to tvt/dmlab/latent_information_acquisition.lua diff --git a/conspecfunction/tvt/dmlab/latent_information_acquisition_factory.lua b/tvt/dmlab/latent_information_acquisition_factory.lua similarity index 100% rename from conspecfunction/tvt/dmlab/latent_information_acquisition_factory.lua rename to tvt/dmlab/latent_information_acquisition_factory.lua diff --git a/conspecfunction/tvt/dmlab/passive_visual_match.lua b/tvt/dmlab/passive_visual_match.lua similarity index 100% rename from conspecfunction/tvt/dmlab/passive_visual_match.lua rename to tvt/dmlab/passive_visual_match.lua diff --git a/conspecfunction/tvt/dmlab/two_keys_to_choose_factory.lua b/tvt/dmlab/two_keys_to_choose_factory.lua similarity index 100% rename from conspecfunction/tvt/dmlab/two_keys_to_choose_factory.lua rename to tvt/dmlab/two_keys_to_choose_factory.lua diff --git a/conspecfunction/tvt/dmlab/two_negative_keys.lua b/tvt/dmlab/two_negative_keys.lua similarity index 100% rename from conspecfunction/tvt/dmlab/two_negative_keys.lua rename to tvt/dmlab/two_negative_keys.lua diff --git a/conspecfunction/tvt/dmlab/visual_match_factory.lua b/tvt/dmlab/visual_match_factory.lua similarity index 100% rename from conspecfunction/tvt/dmlab/visual_match_factory.lua rename to tvt/dmlab/visual_match_factory.lua diff --git a/conspecfunction/tvt/images/avm_notvt.png b/tvt/images/avm_notvt.png similarity index 100% rename from conspecfunction/tvt/images/avm_notvt.png rename to tvt/images/avm_notvt.png diff --git a/conspecfunction/tvt/images/avm_notvt_gamma1.png b/tvt/images/avm_notvt_gamma1.png similarity index 100% rename from conspecfunction/tvt/images/avm_notvt_gamma1.png rename to tvt/images/avm_notvt_gamma1.png diff --git a/conspecfunction/tvt/images/avm_tvt.png b/tvt/images/avm_tvt.png similarity index 100% rename from conspecfunction/tvt/images/avm_tvt.png rename to tvt/images/avm_tvt.png diff --git a/conspecfunction/tvt/images/ktd_notvt.png b/tvt/images/ktd_notvt.png similarity index 100% rename from conspecfunction/tvt/images/ktd_notvt.png rename to tvt/images/ktd_notvt.png diff --git a/conspecfunction/tvt/images/ktd_notvt_gamma1.png b/tvt/images/ktd_notvt_gamma1.png similarity index 100% rename from conspecfunction/tvt/images/ktd_notvt_gamma1.png rename to tvt/images/ktd_notvt_gamma1.png diff --git a/conspecfunction/tvt/images/ktd_tvt.png b/tvt/images/ktd_tvt.png similarity index 100% rename from conspecfunction/tvt/images/ktd_tvt.png rename to tvt/images/ktd_tvt.png diff --git a/conspecfunction/tvt/nest_utils.py b/tvt/nest_utils.py similarity index 100% rename from conspecfunction/tvt/nest_utils.py rename to tvt/nest_utils.py diff --git a/conspecfunction/tvt/pycolab/common.py b/tvt/pycolab/common.py similarity index 100% rename from conspecfunction/tvt/pycolab/common.py rename to tvt/pycolab/common.py diff --git a/conspecfunction/tvt/pycolab/env.py b/tvt/pycolab/env.py similarity index 100% rename from conspecfunction/tvt/pycolab/env.py rename to tvt/pycolab/env.py diff --git a/conspecfunction/tvt/pycolab/game.py b/tvt/pycolab/game.py similarity index 100% rename from conspecfunction/tvt/pycolab/game.py rename to tvt/pycolab/game.py diff --git a/conspecfunction/tvt/pycolab/human_player.py b/tvt/pycolab/human_player.py similarity index 100% rename from conspecfunction/tvt/pycolab/human_player.py rename to tvt/pycolab/human_player.py diff --git a/conspecfunction/tvt/pycolab/key_to_door2.py b/tvt/pycolab/key_to_door2.py similarity index 100% rename from conspecfunction/tvt/pycolab/key_to_door2.py rename to tvt/pycolab/key_to_door2.py diff --git a/conspecfunction/tvt/pycolab/key_to_door3.py b/tvt/pycolab/key_to_door3.py similarity index 100% rename from conspecfunction/tvt/pycolab/key_to_door3.py rename to tvt/pycolab/key_to_door3.py diff --git a/conspecfunction/tvt/pycolab/key_to_door4.py b/tvt/pycolab/key_to_door4.py similarity index 100% rename from conspecfunction/tvt/pycolab/key_to_door4.py rename to tvt/pycolab/key_to_door4.py diff --git a/conspecfunction/tvt/pycolab/objects.py b/tvt/pycolab/objects.py similarity index 100% rename from conspecfunction/tvt/pycolab/objects.py rename to tvt/pycolab/objects.py