get_diffs.py

#	Authored: Max Pfeiffer - 2018
#
#	Reads two .csv files generated by Comma Ai's Cabana and produces 
#	outputs representing the new, and missing messages. Common messages are 
# 	removed from both outputs.
#
# 	Software is free and open to anyone who wishes to use it under the GNU public license 
#
#	Usage: python get_diffs.py "control_filename.csv" "test_filename.csv"
#
#	Result:	new_log.csv and lost_log.csv in the working directory.
 
#!/usr/bin/python
import sys 
import csv

# tracking the unique ids from the control file and the variant file 
control_ids = []
variant_ids = []
lost_ids = []
new_ids = []


# Populate the list of control ids from the first argument infile 
def populate_control_ids():
	control = open(sys.argv[1]) 
	control_csv = csv.reader(control, delimiter = ",", quotechar = "|")
	header = 1 
	for row in control_csv:
		if header:
			header = 0
		else: 
			arb_id = row.pop(1)
			bus = row.pop(1)
			entry = [arb_id, bus] 
			if entry not in control_ids:
				control_ids.append(entry)

	control.close()


# Populate the list of variant ids from second argument infile
def populate_variant_ids():
	variant = open(sys.argv[2])
	variant_csv = csv.reader(variant, delimiter = ",", quotechar = "|" )
	header = 1
	for row in variant_csv:
		if header:
			header = 0
		else: 
			arb_id = row.pop(1)
			bus = row.pop(1)
			entry = [arb_id, bus]
			if entry not in variant_ids: 
				variant_ids.append(entry)

	variant.close()

# generate a list of ids that are preseint in the control, but not in the variant 
def detect_lost_ids():
	for i in range(0,len(control_ids)):
		entry = control_ids[i]
		if entry not in variant_ids:
			lost_ids.append(entry)

# generate a list of ids that at present in the variant and not in the control 
def detect_new_ids():
	for i in range(0,len(variant_ids)):
		entry = variant_ids[i]
		if entry not in control_ids:
			new_ids.append(entry)


# saves the new messages to the output file 
def save_new_messages():
	new_log = open("new_log.csv", "w+")							
	new_writter = csv.writer(new_log, delimiter = ",")
	new_writter.writerow(["time","addr","bus","data"])  

	# re-open the variant file 
	variant = open(sys.argv[2])
	variant_csv = csv.reader(variant, delimiter = ",", quotechar = "|" )

	header = 1
	for row in variant_csv:
		if header:
			header = 0
		else: 
			output = []
			output.append(row.pop(0)) 	
			arb_id = row.pop(0)			
			bus = row.pop(0)			
			entry = [arb_id, bus]		
			if entry in new_ids:
				output.append(arb_id)
				output.append(bus)
				output.append(row.pop(0))
				new_writter.writerow(output) 

	variant.close()
	new_log.close()

# saves the lost messages to the output file 
def save_lost_messages():
	lost_log = open("lost_log.csv", "w+")							
	lost_writter = csv.writer(lost_log, delimiter = ",")
	lost_writter.writerow(["time","addr","bus","data"])  

	# re-open the control file 
	control = open(sys.argv[1])
	control_csv = csv.reader(control, delimiter = ",")

	header = 1
	for row in control_csv:
		if header:
			header = 0
		else: 
			output = []
			output.append(row.pop(0))
			arb_id = row.pop(0)
			bus = row.pop(0)
			entry = [arb_id, bus]
			if entry in lost_ids:
				output.append(arb_id)
				output.append(bus)
				if(len(row)):
					output.append(row.pop(0))
				lost_writter.writerow(output)

	control.close()
	lost_log.close()

# generate a comprehensive list of the IDS in both files, as list_control and list_variant
populate_control_ids()
populate_variant_ids()

# now we want to generate the id lists for the specific output files 
detect_lost_ids()
detect_new_ids()

# having generated these lists, lets create the desired files 
save_new_messages()
save_lost_messages()