From b490dde03f19d56f0fce90173c715c179e25b1b8 Mon Sep 17 00:00:00 2001
From: Dick Pluim <67138922+pluim003@users.noreply.github.com>
Date: Thu, 6 Oct 2022 07:56:50 +0200
Subject: [PATCH] Added 2 tables

To solve the issue mentioned in https://github.com/stevejenkins/pihole-cloudsync/issues/44 had to add the export and import of 2 tables. Also added the -v-flag during import so you get some information on the screen.
---
 pihole-cloudsync | 202 ++++++++++++++++++++++++++---------------------
 1 file changed, 110 insertions(+), 92 deletions(-)

diff --git a/pihole-cloudsync b/pihole-cloudsync
index 08555d9..953a768 100755
--- a/pihole-cloudsync
+++ b/pihole-cloudsync
@@ -7,8 +7,8 @@
 # Steve Jenkins (stevejenkins.com)
 # https://github.com/stevejenkins/pihole-cloudsync
 
-version='5.0'
-update='December 26, 2020'
+version='6.0'
+update='October 6, 2022'
 
 # SETUP
 # Follow the instructions in the README to set up your own private Git
@@ -18,12 +18,12 @@ update='December 26, 2020'
 # USAGE: pihole-cloudsync <option>
 
 # OPTIONS:
-#  --initpush				Initialize Primary Pi-hole in "Push" mode
-#  --initpull				Initialize Secondary Pi-hole in "Pull" mode
-#  --push, --upload, --up, -u		Push (upload) your Pi-hole lists to a remote Git repo
-#  --pull, --download, --down, -d	Pull (download) your lists from a remote Git repo
-#  --help, -h, -?			Show the current version of pihole-cloudsync
-#  --version, -v			Show version number
+#  --initpush                           Initialize Primary Pi-hole in "Push" mode
+#  --initpull                           Initialize Secondary Pi-hole in "Pull" mode
+#  --push, --upload, --up, -u           Push (upload) your Pi-hole lists to a remote Git repo
+#  --pull, --download, --down, -d       Pull (download) your lists from a remote Git repo
+#  --help, -h, -?                       Show the current version of pihole-cloudsync
+#  --version, -v                        Show version number
 
 # EXAMPLES:
 #  'pihole-cloudsync --push' will push (upload) your lists to a remote Git repo
@@ -37,8 +37,11 @@ pihole_dir='/etc/pihole'
 gravity_db='/etc/pihole/gravity.db'
 dnsmasq_dir='/etc/dnsmasq.d/'
 ad_list='adlist.csv'
+ad_by_group_list='adlist_by_group.csv'
 custom_list='custom.list'
 domain_list='domainlist.csv'
+domain_by_group_list='domainlist_by_group.csv'
+
 cname_list='05-pihole-custom-cname.conf'
 ###########################################################################
 # SHOULDN'T NEED TO EDIT BELOW THIS LINE
@@ -51,113 +54,128 @@ fi
 
 # FUNCTIONS
 push_initialize () {
-	# Go to Pi-hole directory, exit if doesn't exist
-	cd $pihole_dir || exit
+        # Go to Pi-hole directory, exit if doesn't exist
+        cd $pihole_dir || exit
 
-	# Verify Custom and CNAME lists exist
-	$SUDO touch $custom_list
-	$SUDO touch $dnsmasq_dir/$cname_list
+        # Verify Custom and CNAME lists exist
+        $SUDO touch $custom_list
+        $SUDO touch $dnsmasq_dir/$cname_list
 
-	# Copy local Custom and CNAME lists to local Git repo
-	$SUDO cp $custom_list $personal_git_dir
-	$SUDO cp $dnsmasq_dir/$cname_list $personal_git_dir
+        # Copy local Custom and CNAME lists to local Git repo
+        $SUDO cp $custom_list $personal_git_dir
+        $SUDO cp $dnsmasq_dir/$cname_list $personal_git_dir
 
-	# Go to local Git repo directory
-	cd $personal_git_dir || exit
+        # Go to local Git repo directory
+        cd $personal_git_dir || exit
 
-	# Export Ad and Domain lists from Gravity database
-	$SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM adlist" >$ad_list
-	$SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM domainlist" >$domain_list
+        # Export Ad and Domain lists from Gravity database
+        $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM adlist" >$ad_list
+        $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM adlist_by_group" >$ad_by_group_list
+        $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM domainlist" >$domain_list
+        $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM domainlist_by_group" >$domain_by_group_list
 
-	# Add all lists to local Git repo
-	$SUDO git add .
-	echo "Local Pi-hole initialized in Push mode and local lists were added to local Git repo. Run 'pihole-cloudsync --push' to push to remote Git repo.";
+        # Add all lists to local Git repo
+        $SUDO git add .
+        echo "Local Pi-hole initialized in Push mode and local lists were added to local Git repo. Run 'pihole-cloudsync --push' to push to remote Git repo.";
 }
 pull_initialize () {
-	# Go to Pi-hole directory, exit if doesn't exist
-	cd $personal_git_dir || exit
+        # Go to Pi-hole directory, exit if doesn't exist
+        cd $personal_git_dir || exit
 
-	# Update local Git repo from remote Git repo
-	$SUDO git remote update > /dev/null
+        # Update local Git repo from remote Git repo
+        $SUDO git remote update > /dev/null
 
-	# Remove -q option if you don't want to run in "quiet" mode
-	$SUDO git fetch --all -q
-	$SUDO git reset --hard origin/master -q
+        # Remove -q option if you don't want to run in "quiet" mode
+        $SUDO git fetch --all -q
+        $SUDO git reset --hard origin/master -q
 
-	# Stop DNS server
-	$SUDO service pihole-FTL stop
+        # Stop DNS server
+        $SUDO service pihole-FTL stop
 
-	# Overwrite local files
-	$SUDO cp $custom_list $pihole_dir
-	$SUDO cp $cname_list $dnsmasq_dir
+        # Overwrite local files
+        $SUDO cp $custom_list $pihole_dir
+        $SUDO cp $cname_list $dnsmasq_dir
 
-	# Overwrite local database tables
-	$SUDO sqlite3 $gravity_db "DROP TABLE adlist;"
-	$SUDO sqlite3 $gravity_db -header -csv ".import adlist.csv adlist"
-	$SUDO sqlite3 $gravity_db "DROP TABLE domainlist;"
-	$SUDO sqlite3 $gravity_db -header -csv ".import domainlist.csv domainlist"
+        # Overwrite local database tables
+        $SUDO sqlite3 $gravity_db "DROP TABLE adlist;"
+        $SUDO sqlite3 $gravity_db -header -csv ".import adlist.csv adlist -v"
+        $SUDO sqlite3 $gravity_db "DROP TABLE adlist_by_group;"
+        $SUDO sqlite3 $gravity_db -header -csv ".import adlist_by_group.csv adlist_by_group -v"
+        $SUDO sqlite3 $gravity_db "DROP TABLE domainlist;"
+        $SUDO sqlite3 $gravity_db -header -csv ".import domainlist.csv domainlist -v"
+        $SUDO sqlite3 $gravity_db "DROP TABLE domainlist_by_group;"
+        $SUDO sqlite3 $gravity_db -header -csv ".import domainlist_by_group.csv domainlist_by_group -v"
 
-	# Restart Pi-hole to pick up changes
-	$SUDO pihole -g
+        # Restart Pi-hole to pick up changes
+        $SUDO pihole -g
 
-	# Display success messages
-	echo "Local Pi-hole initialized in Pull mode and first pull successfully completed.";
-	echo "Future pulls can now be perfomed with 'pihole-cloudsync --pull'.";
+        # Run another time due to foreign key mismatch-errors
+        #$SUDO pihole -g
+
+        # Display success messages
+        echo "Local Pi-hole initialized in Pull mode and first pull successfully completed.";
+        echo "Future pulls can now be perfomed with 'pihole-cloudsync --pull'.";
 }
 push () {
-	 # Go to Pi-hole directory, exit if doesn't exist
-	cd $pihole_dir || exit
+         # Go to Pi-hole directory, exit if doesn't exist
+        cd $pihole_dir || exit
 
         # Copy local Custom and CNAME lists to local Git repo
         $SUDO cp $custom_list $personal_git_dir
         $SUDO cp $dnsmasq_dir/$cname_list $personal_git_dir
 
-	# Go to local Git repo directory
+        # Go to local Git repo directory
         cd $personal_git_dir || exit
 
         # Export Ad and Domain lists from Gravity database
         $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM adlist" >$ad_list
         $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM domainlist" >$domain_list
+        $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM adlist_by_group" >$ad_by_group_list
+        $SUDO sqlite3 $gravity_db -header -csv "SELECT * FROM domainlist_by_group" >$domain_by_group_list
 
-	# Compare local files to remote Git repo
-	$SUDO git remote update > /dev/null
+        # Compare local files to remote Git repo
+        $SUDO git remote update > /dev/null
 
-	# If local files are different than remote, update remote Git repo
+        # If local files are different than remote, update remote Git repo
         CHANGED=$($SUDO git --work-tree=$personal_git_dir status --porcelain)
         if [ -n "${CHANGED}" ]; then
                 echo 'Local Pi-hole lists are different than remote Git repo. Updating remote repo...';
-		rightnow=$(date +"%B %e, %Y %l:%M%p")
-		# Remove -q option if you don't want to run in "quiet" mode
-		$SUDO git commit -a -m "Updated $rightnow" -q
-		$SUDO git push -q
-		echo 'Done!';
-		exit 0
+                rightnow=$(date +"%B %e, %Y %l:%M%p")
+                # Remove -q option if you don't want to run in "quiet" mode
+                $SUDO git commit -a -m "Updated $rightnow" -q
+                $SUDO git push -q
+                echo 'Done!';
+                exit 0
         else
-	# If local files are the same as remote, do nothing and exit
-	echo 'Remote Git repo matches local Pi-hole lists. No further action required.';
-		exit 0
+        # If local files are the same as remote, do nothing and exit
+        echo 'Remote Git repo matches local Pi-hole lists. No further action required.';
+                exit 0
         fi
 }
 pull () {
         # Go to Pi-hole directory, exit if doesn't exist
-	cd $personal_git_dir || exit
+        cd $personal_git_dir || exit
 
-	# Update local Git repo from remote Git repo
-	$SUDO git remote update > /dev/null
-	CHANGED=$($SUDO git log HEAD..origin/master --oneline)
-	if [ -n "${CHANGED}" ]; then
+        # Update local Git repo from remote Git repo
+        $SUDO git remote update > /dev/null
+        CHANGED=$($SUDO git log HEAD..origin/master --oneline)
+        if [ -n "${CHANGED}" ]; then
                 echo 'Remote Git repo is different than local Pi-hole lists. Updating local lists...';
                 # Remove -q option if you don't want to run in "quiet" mode
                 $SUDO git fetch --all -q
-		$SUDO git reset --hard origin/master -q
+                $SUDO git reset --hard origin/master -q
                 $SUDO service pihole-FTL stop
                 $SUDO cp $custom_list $pihole_dir
                 $SUDO cp $cname_list $dnsmasq_dir
                 $SUDO sqlite3 $gravity_db "DROP TABLE adlist;"
-                $SUDO sqlite3 $gravity_db -header -csv ".import adlist.csv adlist"
+                $SUDO sqlite3 $gravity_db -header -csv ".import adlist.csv adlist -v"
                 $SUDO sqlite3 $gravity_db "DROP TABLE domainlist;"
-                $SUDO sqlite3 $gravity_db -header -csv ".import domainlist.csv domainlist"
-		$SUDO pihole -g
+                $SUDO sqlite3 $gravity_db -header -csv ".import domainlist.csv domainlist -v"
+                $SUDO sqlite3 $gravity_db "DROP TABLE adlist_by_group;"
+                $SUDO sqlite3 $gravity_db -header -csv ".import adlist_by_group.csv adlist_by_group -v"
+                $SUDO sqlite3 $gravity_db "DROP TABLE domainlist_by_group;"
+                $SUDO sqlite3 $gravity_db -header -csv ".import domainlist_by_group.csv domainlist_by_group -v"
+                $SUDO pihole -g
                 echo 'Done!';
                 exit 0
         else
@@ -178,40 +196,40 @@ do
     # Initialize - adds primary Pi-hole's lists to local Git repo before first push/upload
     if [ "$arg" == "--initpush" ]
     then
-	echo "$arg option detected. Initializing local Git repo for Push/Upload.";
-	push_initialize
-	exit 0
+        echo "$arg option detected. Initializing local Git repo for Push/Upload.";
+        push_initialize
+        exit 0
     # Initialize - adds primary Pi-hole's lists to local Git repo before first push/upload
     elif [ "$arg" == "--initpull" ]
     then
-	echo "$arg option detected. Initializing local Git repo for Pull/Download.";
-	pull_initialize
-	exit 0
+        echo "$arg option detected. Initializing local Git repo for Pull/Download.";
+        pull_initialize
+        exit 0
     # Push / Upload - Pushes updated local Pi-hole lists to remote Git repo
     elif [ "$arg" == "--push" ] || [ "$arg" == "--upload" ] || [ "$arg" == "--up" ] || [ "$arg" == "-u" ]
     then
-	echo "$arg option detected. Running in Push/Upload mode."
-	push
-	exit 0
+        echo "$arg option detected. Running in Push/Upload mode."
+        push
+        exit 0
     # Pull / Download - Pulls updated Pi-hole lists from remote Git repo
     elif [ "$arg" == "--pull" ] || [ "$arg" == "--download" ] || [ "$arg" == "--down" ]|| [ "$arg" == "-d" ]
     then
         echo "$arg option detected. Running in Pull/Download mode."
-	pull
+        pull
         exit 0
     # Help - Displays help dialog
     elif [ "$arg" == "--help" ] || [ "$arg" == "-h" ] || [ "$arg" == "-?" ]
     then
-	cat << EOF
+        cat << EOF
 Usage: pihole-cloudsync <option>
 
 Options:
-  --push, --upload, --up, -u		Push (upload) your Pi-hole lists to a remote Git repo
-  --pull, --download, --down, -d	Pull (download) your lists from a remote Git repo
-  --initpush				Initialize Primary Pi-hole in "Push" mode
-  --initpull				Initialize Secondary Pi-hole in "Pull" mode
-  --help, -h, -?			Show this help dialog
-  --version, -v				Show the current version of pihole-cloudsync
+  --push, --upload, --up, -u            Push (upload) your Pi-hole lists to a remote Git repo
+  --pull, --download, --down, -d        Pull (download) your lists from a remote Git repo
+  --initpush                            Initialize Primary Pi-hole in "Push" mode
+  --initpull                            Initialize Secondary Pi-hole in "Pull" mode
+  --help, -h, -?                        Show this help dialog
+  --version, -v                         Show the current version of pihole-cloudsync
 
 Examples:
   'pihole-cloudsync --push' will push (upload) your lists to a Git repo
@@ -222,13 +240,13 @@ EOF
 
     # Version - Displays version number
     elif [ "$arg" == "--version" ] || [ "$arg" == "-v" ]
-	then
-	echo 'pihole-cloudsync v'$version' - Updated '"$update";
-	echo 'https://github.com/stevejenkins/pihole-cloudsync';
+        then
+        echo 'pihole-cloudsync v'$version' - Updated '"$update";
+        echo 'https://github.com/stevejenkins/pihole-cloudsync';
 
     # Invalid command line option was passed
     else
-	echo "Invalid command line option. Try --push, --pull, or --help."
-	exit 1
+        echo "Invalid command line option. Try --push, --pull, or --help."
+        exit 1
     fi
 done