Visualization of Predicted Trajectories

ynet/evaluate_inD_longterm.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[2]:
+
+
+import pandas as pd
+import yaml
+import argparse
+import torch
+from model import YNet
+
+
+# In[3]:
+
+
+get_ipython().run_line_magic('load_ext', 'autoreload')
+get_ipython().run_line_magic('autoreload', '2')
+
+
+# #### Some hyperparameters and settings
+
+# In[4]:
+
+
+CONFIG_FILE_PATH = 'config/inD_longterm.yaml'  # yaml config file containing all the hyperparameters
+DATASET_NAME = 'ind'
+
+TEST_DATA_PATH = 'data/inD/test.pkl'
+TEST_IMAGE_PATH = 'data/inD/test'
+OBS_LEN = 5  # in timesteps
+PRED_LEN = 30  # in timesteps
+NUM_GOALS = 20  # K_e
+NUM_TRAJ = 1  # K_a
+
+ROUNDS = 3  # Y-net is stochastic. How often to evaluate the whole dataset
+BATCH_SIZE = 8
+
+
+# #### Load config file and print hyperparameters
+
+# In[5]:
+
+
+with open(CONFIG_FILE_PATH) as file:
+    params = yaml.load(file, Loader=yaml.FullLoader)
+experiment_name = CONFIG_FILE_PATH.split('.yaml')[0].split('config/')[1]
+params
+
+
+# #### Load preprocessed Data
+
+# In[6]:
+
+
+df_test = pd.read_pickle(TEST_DATA_PATH)
+
+
+# In[7]:
+
+
+df_test.head()
+
+
+# #### Initiate model and load pretrained weights
+
+# In[8]:
+
+
+model = YNet(obs_len=OBS_LEN, pred_len=PRED_LEN, params=params)
+
+
+# In[9]:
+
+
+model.load(f'pretrained_models/{experiment_name}_weights.pt')
+
+
+# #### Evaluate model
+
+# In[10]:
+
+
+model.evaluate(df_test, params, image_path=TEST_IMAGE_PATH,
+               batch_size=BATCH_SIZE, rounds=ROUNDS, 
+               num_goals=NUM_GOALS, num_traj=NUM_TRAJ, device=None, dataset_name=DATASET_NAME)
+
+
+# In[1]:
+
+
+get_ipython().system('nvidia-smi')
+
+
+# In[ ]:
+
+
+
+
+
+# In[ ]:
+
+
+
+
+
+# In[ ]:
+
+
+
+

ynet/test.py

@@ -1,7 +1,8 @@
 import torch
 import torch.nn as nn
-from utils.image_utils import get_patch, sampling, image2world
+from utils.image_utils import get_patch, sampling, image2world, plot_results
 from utils.kmeans import kmeans
+import skimage.io
 
 
 def torch_multivariate_gaussian_heatmap(coordinates, H, W, dist, sigma_factor, ratio, device, rot=False):
@@ -55,7 +56,7 @@ def evaluate(model, val_loader, val_images, num_goals, num_traj, obs_len, batch_
 	:param mode: ['val', 'test']
 	:return: val_ADE, val_FDE for one epoch
 	"""
-
+	im = skimage.io.imread('data/inD/test/scene1/reference.png')
 	model.eval()
 	val_ADE = []
 	val_FDE = []
@@ -212,6 +213,12 @@ def evaluate(model, val_loader, val_images, num_goals, num_traj, obs_len, batch_
 
 				val_FDE.append(((((gt_goal - waypoint_samples[:, :, -1:]) / resize) ** 2).sum(dim=3) ** 0.5).min(dim=0)[0])
 				val_ADE.append(((((gt_future - future_samples) / resize) ** 2).sum(dim=3) ** 0.5).mean(dim=2).min(dim=0)[0])
+
+				plot_results(gt_future, future_samples, observed, scene_image, val_images[scene], resize, with_bg=False)
+# 				plot_results(gt_future, future_samples, observed, scene_image, val_images[scene], resize, with_bg=True, save_path='viz/'+str(scene)+'_'+str(counter)+'.png')
+				counter+=1
+
+				# plt.savefig()
 
 		val_ADE = torch.cat(val_ADE).mean()
 		val_FDE = torch.cat(val_FDE).mean()

ynet/utils/image_utils.py

@@ -2,6 +2,9 @@
 import torch
 import cv2
 import torch.nn.functional as F
+import matplotlib.pyplot as plt
+import skimage.io
+from skimage.transform import rescale, resize, downscale_local_mean
 
 def gkern(kernlen=31, nsig=4):
 	"""	creates gaussian kernel with side length l and a sigma of sig """
@@ -135,3 +138,20 @@ def image2world(image_coords, scene, homo_mat, resize):
 	traj_image2world = traj_image2world[:, :2]
 	traj_image2world = traj_image2world.view_as(image_coords)
 	return traj_image2world
+
+def plot_results(gt_future, future_samples, observed, scene_image, im, resize, with_bg=True, save_path=None):
+	plt.scatter(gt_future.cpu()[1,:,0]/resize, gt_future.cpu()[1,:,1]/resize, label='ground truth', zorder=3)
+	plt.scatter(future_samples.cpu()[:,1,:,0]/resize, future_samples.cpu()[:,1,:,1]/resize, label='predictions', alpha=0.1, zorder=2)
+	plt.scatter(observed[5:10,0]/resize, observed[5:10,1]/resize, label='observed_past', color='cyan', zorder=1)
+	scene_image_rescaled = rescale(scene_image.cpu().squeeze()[1].squeeze(), 1/resize)
+	im_rescaled = rescale(im.cpu().squeeze()[1].squeeze(), 1/resize)
+	plt.imshow(scene_image_rescaled, alpha=0.001)
+	if with_bg:
+		plt.imshow(scene_image_rescaled)
+		plt.imshow(im_rescaled, alpha=0.7)
+	plt.legend()							
+
+	if save_path is not None:
+		plt.savefig(save_path)
+	plt.show()
+