Added sitemap an 404 logs import

.gitignore

@@ -0,0 +1,46 @@
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+Icon?
+ehthumbs.db
+Thumbs.db
+
+# PHP / WEB STORM #
+###################
+.idea/
+
+# SASS  #
+#########
+.sass-cache/

Redirect.php

@@ -0,0 +1,215 @@
+<?php if (!defined('TL_ROOT')) die('You can not access this file directly!');
+
+/**
+ * @copyright 	mediabakery 2013
+ * @author 			Sebastian Tilch <s.tilch@mediabakery.de>
+ * @license    	LGPL
+ * @package 		redirect4ward
+ * @filesource
+ */
+
+class Redirect extends Backend
+{
+
+		private $arrHosts;
+
+		private $arrAddedUrls;
+
+		public function __construct()
+		{
+				parent::__construct();
+				$this->Template = new BackendTemplate('be_import_sitemap');
+				$this->arrAddedUrls = array();
+		}
+
+		/**
+		 * Import a sitemap
+		 */
+		public function importSitemap()
+		{
+				$blnError = false;
+				$this->loadLanguageFile('tl_redirect4ward');
+				$this->blnSave = true;
+
+				$this->Template->sitemap_source = $this->getSitemapWidget();
+				$this->Template->sitemap_source_label = $GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_source_label'];
+
+				$this->Template->hrefBack = ampersand(str_replace('&key=importSitemap', '', $this->Environment->request));
+				$this->Template->goBack = $GLOBALS['TL_LANG']['MSC']['goBack'];
+				$this->Template->headline = $GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_headline'];
+				$this->Template->request = ampersand($this->Environment->request, ENCODE_AMPERSANDS);
+				$this->Template->submit = specialchars($GLOBALS['TL_LANG']['MSC']['continue']);
+
+				if($this->Input->post('FORM_SUBMIT') == 'tl_import_sitemap' && $this->blnSave)
+				{
+						$this->import('tl_redirect4ward');
+						$this->import('Request');
+
+						$this->arrHosts = $this->tl_redirect4ward->getHosts();
+						$strSource = $this->Template->sitemap_source->value;
+						if(!strlen($strSource))
+						{
+								$this->setStatus(sprintf($GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapErrorMessage'], $strSource, $strSource), 'Redirect importSitemap()', true);
+								$blnError = true;
+
+						}
+						$intCounter = 0;
+
+						$objReq = new Request();
+						$objReq->send($strSource);
+
+						if(!$objReq->hasError())
+						{
+								$objXml = simplexml_load_string($objReq->response);
+								if(!$objXml)
+								{
+										$this->setStatus(sprintf($GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapErrorMessage'], $strSource, $strSource), 'Redirect importSitemap()', true);
+										$blnError = true;
+								}
+								foreach($objXml as $objUrl)
+								{
+										if($this->addLocation($objUrl->loc))
+												$intCounter++;
+								}
+						}
+						else
+						{
+								$this->setStatus(sprintf($GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapErrorMessage'], $strSource, $strSource), 'Redirect importSitemap()', true);
+								$blnError = true;
+						}
+						if(!$blnError)
+								$this->setStatus(sprintf($GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapMessage'], $intCounter, $strSource, $strSource), 'Redirect importSitemap()');
+				}
+
+				return $this->Template->parse();
+		}
+
+		/**
+		 * Import from logs
+		 */
+		public function importLogs()
+		{
+				$this->import('tl_redirect4ward');
+
+				$this->arrHosts = $this->tl_redirect4ward->getHosts();
+
+				$intStart = time() - 3600;
+
+				$objLogs = $this->Database->prepare("SELECT text FROM tl_log WHERE tstamp > ? AND func = 'PageError404 generate()'")->execute($intStart);
+				$intCounter = 0;
+				while($objLogs->next())
+				{
+						preg_match('/\((.*)\)/', $objLogs->text, $arrUrls);
+						if($this->addLocation($arrUrls[1]))
+								$intCounter++;
+				}
+				$this->setStatus(sprintf($GLOBALS['TL_LANG']['tl_redirect4ward']['addLogMessage'], $intCounter, $strSource, $strSource), 'Redirect importLogs()');
+				$this->redirect(ampersand(str_replace('&key=importLogs', '', $this->Environment->request)), 301);
+		}
+
+		private function addLocation($strLoc)
+		{
+				$arrUrlFragments = parse_url($strLoc);
+				$strUrl = trim(preg_replace('/^' . preg_quote($GLOBALS['TL_CONFIG']['websitePath'], '/') . '/', '', $arrUrlFragments['path']), '/');
+
+				// If the url is "index.php" add the query
+				if ($strUrl == 'index.php' && strlen($arrUrlFragments['query']))
+				{
+						$strUrl = 'index.php?' . $arrUrlFragments['query'];
+				}
+
+				$arrSet = array(
+						'url' => $strUrl,
+						'host' => str_replace('www.', '', $arrUrlFragments['host'])
+				);
+				return $this->addSite($arrSet);
+		}
+
+		private function addSite($arrSet)
+		{
+
+				if(!strlen($arrSet['url']))
+						return FALSE;
+				if(!in_array($arrSet['host'], $this->arrHosts))
+				{
+						unset($arrSet['host']);
+				}
+
+				$arrSet['tstamp'] = time();
+				$objRedirect = $this->Database->prepare("SELECT id FROM tl_redirect4ward WHERE url=? AND host=?")->limit(1)->execute($arrSet['url'], isset($arrSet['host']) ? $arrSet['host'] : '');
+
+				if($objRedirect->numRows == 0 && !array_key_exists(static::getHostUrlHash($arrSet), $this->arrAddedUrls))
+				{
+						$this->arrAddedUrls[static::getHostUrlHash($arrSet)] = true;
+						$this->Database->prepare("INSERT INTO tl_redirect4ward %s")->set($arrSet)->execute();
+						return TRUE;
+				}
+				return FALSE;
+		}
+
+		private static function getHostUrlHash($arrSet)
+		{
+				return standardize($arrSet['host'].'/'.$arrSet['url']);
+		}
+
+		private function setStatus($strStatus, $strFunction, $blnError = false)
+		{
+				if(version_compare(VERSION, '2.11') >= 0)
+				{
+						// addInfoMessage and addErrorMessage are available
+						if($blnError)
+						{
+								$this->addErrorMessage($strStatus);
+						}
+						else
+						{
+								$this->addInfoMessage($strStatus);
+						}
+				}
+
+				if($blnError)
+				{
+						$this->log(strip_tags($strStatus), $strFunction, TL_ERROR);
+						$this->Template->statusClass = 'tl_error';
+				}
+				else
+				{
+						$this->log(strip_tags($strStatus), $strFunction, TL_INFO);
+						$this->Template->statusClass = 'tl_info';
+				}
+				$this->Template->hasStatus = true;
+				$this->Template->status = $strStatus;
+
+		}
+
+		private function getSitemapWidget($value = null)
+		{
+				$widget = new TextField();
+
+				$widget->id 				= 'sitemap_source';
+				$widget->name 			= 'sitemap_source';
+				$widget->mandatory 	= true;
+				$widget->strTable 	= 'tl_redirect4ward';
+				$widget->strField 	= 'sitemap_source';
+				$widget->value 			= $value;
+				$widget->label 			= $GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_source'][0];
+
+				if($GLOBALS['TL_CONFIG']['showHelp'] && strlen($GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_source'][1]))
+				{
+						$widget->help = $GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_source'][1];
+				}
+
+				// Validate input
+				if($this->Input->post('FORM_SUBMIT') == 'tl_import_sitemap')
+				{
+						$widget->validate();
+						if($widget->hasErrors())
+						{
+								$this->blnSave = false;
+						}
+				}
+
+				return $widget;
+		}
+
+}

config/config.php

@@ -13,8 +13,10 @@
 
 array_insert($GLOBALS['BE_MOD']['system'],2,array(
 	'redirect4ward' => array(
-		'tables'     => array('tl_redirect4ward'),
-		'icon' => 'system/modules/redirect4ward/html/icon.png' 
+		'tables'     		=> array('tl_redirect4ward'),
+		'importSitemap' => array('Redirect','importSitemap'),
+		'importLogs' 		=> array('Redirect','importLogs'),
+		'icon' 					=> 'system/modules/redirect4ward/html/icon.png'
 	)
 ));
 
@@ -24,5 +26,3 @@
 else {
 	$GLOBALS['TL_HTACCESS_SUBMODULES']['rewrite']['redirect4ward'] = 'HtaccessRedirect4ward';
 }
-
-?>

dca/tl_redirect4ward.php

@@ -41,6 +41,18 @@
 		),
 		'global_operations' => array
 		(
+      'importSitemap' => array(
+        'label'								=> &$GLOBALS['TL_LANG']['tl_redirect4ward']['importSitemap'],
+        'href' 								=> 'key=importSitemap',
+				'class'               => 'header_new',
+        'attributes' 					=> 'onclick="Backend.getScrollOffset();"'
+    	),
+      'importLogs' => array(
+        'label' 							=> &$GLOBALS['TL_LANG']['tl_redirect4ward']['importLogs'],
+        'href' 								=> 'key=importLogs',
+				'class'               => 'header_new',
+      	'attributes' 					=> 'onclick="Backend.getScrollOffset();"'
+  		),
 			'all' => array
 			(
 				'label'               => &$GLOBALS['TL_LANG']['MSC']['all'],
@@ -363,7 +375,7 @@ public function getActivePageLanguages()
 
 		foreach ($arrRootPages as $value)
 		{
-			if (key_exists($value['language'], $arrLanguages))
+			if (array_key_exists($value['language'], $arrLanguages))
 			{
 				$arrReturn[$value['language']] = $arrLanguages[$value['language']];
 			}
@@ -429,4 +441,3 @@ public function pagePicker(DataContainer $dc)
 	}
 
 }
-?>

languages/de/tl_redirect4ward.php

@@ -54,10 +54,26 @@
 
 $GLOBALS['TL_LANG']['tl_redirect4ward']['new']    = array('Neue Weiterleitung', 'Eine neue Weiterleitung anlegen');
 
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importSitemap'][0]	= "Sitemap importieren";
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importSitemap'][1]	= "Sitemap importieren.";
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importLogs'][0]	= "404 Logs importieren";
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importLogs'][1]	= "404 Logs der letzten Stunde importieren.";
+
 $GLOBALS['TL_LANG']['tl_redirect4ward']['type_legend'] = 'Weiterleitungs-Typ';
 $GLOBALS['TL_LANG']['tl_redirect4ward']['target_legend'] = 'Ziel';
 $GLOBALS['TL_LANG']['tl_redirect4ward']['expert_legend'] = 'Weitere Einstellungen';
 $GLOBALS['TL_LANG']['tl_redirect4ward']['publish_legend'] = 'Veröffentlichung';
 
+$GLOBALS['TL_LANG']['tl_redirect4ward']['source'][0] = "Sitemap URL";
+$GLOBALS['TL_LANG']['tl_redirect4ward']['source'][1] = "URL der zu importierenden Sitemap.";
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['addLogMessage'] = '%d neue Weiterleitungen aus den Logs hinzugefügt.';
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapMessage'] = '%d neue Weiterleitungen aus der Sitemap (<a href="%s">%s</a>) hinzugefügt.';
+$GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapErrorMessage'] = 'Sitemap (<a href="%s">%s</a>) kann nicht gelesen werden.';
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['everyHost'] = 'jeder Host';
 
-?>
+$GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_source_label'] = 'Sitemap URL';
+$GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_headline'] = 'Sitemap importieren';

languages/en/tl_redirect4ward.php

@@ -53,9 +53,27 @@
 
 
 $GLOBALS['TL_LANG']['tl_redirect4ward']['new']    = array('New redirection', 'Creates a new redirection.');
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importSitemap'][0]	= "Import sitemap";
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importSitemap'][1]	= "Import sitemap.";
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importLogs'][0]	= "Import 404 logs";
+$GLOBALS['TL_LANG']['tl_redirect4ward']['importLogs'][1]	= "Import 404 logs.";
+
 $GLOBALS['TL_LANG']['tl_redirect4ward']['type_legend'] = 'Redirect-type';
 $GLOBALS['TL_LANG']['tl_redirect4ward']['target_legend'] = 'Target';
 $GLOBALS['TL_LANG']['tl_redirect4ward']['expert_legend'] = 'Config';
 $GLOBALS['TL_LANG']['tl_redirect4ward']['publish_legend'] = 'Publish settings';
 
-?>
+$GLOBALS['TL_LANG']['tl_redirect4ward']['source'][0] = "Sitemap URL";
+$GLOBALS['TL_LANG']['tl_redirect4ward']['source'][1] = "Sitemap URL.";
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['addLogMessage'] = 'Add %d new redirections from the log.';
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapMessage'] = 'Add %d new redirection from the sitemap (<a href="%s">%s</a>).';
+$GLOBALS['TL_LANG']['tl_redirect4ward']['addSitemapErrorMessage'] = 'Can not read sitemap (<a href="%s">%s</a>).';
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['everyHost'] = 'jeder Host';
+
+$GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_source_label'] = 'Sitemap URL';
+$GLOBALS['TL_LANG']['tl_redirect4ward']['sitemap_headline'] = 'Import sitemap';