From a20210cc8de810359e913944e2b02484ef422621 Mon Sep 17 00:00:00 2001 From: Sergey Batanov Date: Sun, 19 Mar 2017 11:51:18 +0300 Subject: [PATCH 1/6] Fixed event parser. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser failed when meet comma, bracket or quote in comment. Парсер сбивался, если в комментарии события были символы запятой, фигурной скобки или кавычек. --- EventLogLoaderService/EventLogProcessor.vb | 16 +++++-- EventLogLoaderService/Parser.vb | 53 ++++++++++++++-------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index 4f102c0..1dad816 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -768,7 +768,15 @@ Public Class EventLogProcessor copy.ColumnMappings.Add(jj, jj) Next copy.DestinationTableName = "Events" - copy.WriteToServer(dt) + Try + copy.WriteToServer(dt) + Catch ex As InvalidOperationException + Log.Error("Ошибка сохранения в БД записи по ИБ " + InfobaseName + " : " + ex.Message) + Catch ex As Exception + Log.Error("Ошибка сохранения в БД записи по ИБ " + InfobaseName + " : " + ex.Message) + End Try + 'InvalidOperationException + End Using SaveReadParametersToFile() @@ -1406,11 +1414,11 @@ Public Class EventLogProcessor If ItsEndOfEvent(TextLine, CountBracket, TextBlockOpen) Then NewLine = True - If Not StrEvent Is Nothing Then + If Not StrEvent Is Nothing And (StrEvent.Length > 1) Then Try AddEvent(StrEvent) Catch ex As Exception - + Log.Error(ex.Message) End Try '*** @@ -1447,7 +1455,7 @@ Public Class EventLogProcessor End If Next - ItsEndOfEvent = (Count = 0) + ItsEndOfEvent = (Count = 0) And Not TextBlockOpen End Function diff --git a/EventLogLoaderService/Parser.vb b/EventLogLoaderService/Parser.vb index f3c1959..47ceb45 100644 --- a/EventLogLoaderService/Parser.vb +++ b/EventLogLoaderService/Parser.vb @@ -4,36 +4,53 @@ Dim ArrayLines(0) - Dim Text2 = Text.Substring(1, IIf(Text.EndsWith(","), Text.Length - 3, Text.Length - 2)) + "," + If Text Is Nothing Or Text.Length = 0 Then + Return ArrayLines + End If - Dim Str = "" + ' Сим отсекаем скобочки + Dim Text2 = Text.Substring(1, IIf(Text.EndsWith(","), Text.Length - 3, Text.Length - 2)) + "," - Dim Delim = Text2.IndexOf(",") Dim i = 0 + Dim Str = "" + Dim TextBlockOpen = False + Dim count = 0 + Dim BracketCount = 0 - While Delim > 0 - Str = Str + Text2.Substring(0, Delim).Trim - Text2 = Text2.Substring(Delim + 1) + For i = 0 To Text2.Length - 1 - If CountSubstringInString(Str, "{") = CountSubstringInString(Str, "}") _ - And Math.IEEERemainder(CountSubstringInString(Str, """"), 2) = 0 Then + Dim simb = Text2.Substring(i, 1) - ReDim Preserve ArrayLines(i) + If (simb = ",") And Not TextBlockOpen And (BracketCount = 0) Then - If Str.StartsWith("""") And Str.EndsWith("""") Then - Str = Str.Substring(1, Str.Length - 2) - End If + ReDim Preserve ArrayLines(count) + ArrayLines(count) = Str + count = count + 1 - ArrayLines(i) = Str - i = i + 1 Str = "" + + Else If simb = "{" And Not TextBlockOpen Then + + BracketCount = BracketCount + 1 + Str = Str + simb + + Else If simb = "}" And Not TextBlockOpen Then + + BracketCount = BracketCount - 1 + Str = Str + simb + + Else If simb = """" Then + + TextBlockOpen = Not TextBlockOpen + Str = Str + """" + Else - Str = Str + "," - End If - Delim = Text2.IndexOf(",") + Str = Str + simb + + End If - End While + Next Return ArrayLines From 298a07a19cf1a8c71956d58057cb5de3be9c498c Mon Sep 17 00:00:00 2001 From: Sergey Batanov Date: Sun, 19 Mar 2017 15:00:31 +0300 Subject: [PATCH 2/6] Fixed reference loading. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When loading reference file in case of file format the data was cleared but not filled if the reference file was not changed. В случае работы с файловым журналом справочники очищались, но не перезаполнялись. --- EventLogLoaderService/EventLogProcessor.vb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index 1dad816..18046ae 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -989,7 +989,7 @@ Public Class EventLogProcessor LastProcessedObjectForDebug = TextObject - Dim a = ParserServices.ParseEventLogString(TextObject) + Dim a = ParserServices.ParseEventLogString(TextObject.ToString().Trim()) If Not a Is Nothing Then Select Case a(0) @@ -1029,7 +1029,7 @@ Public Class EventLogProcessor End Sub - Sub LoadReference() + Sub ClearReference() 'Clear all reference dictionaries DictUsers.Clear() @@ -1041,10 +1041,16 @@ Public Class EventLogProcessor DictMainPorts.Clear() DictSecondPorts.Clear() + End Sub + + Sub LoadReference() + Dim FileName = Path.Combine(Catalog, "1Cv8.lgd") If My.Computer.FileSystem.FileExists(FileName) Then + ClearReference() + Try Dim Conn = New SQLite.SQLiteConnection("Data Source=" + FileName) Conn.Open() @@ -1129,6 +1135,7 @@ Public Class EventLogProcessor If FI.LastWriteTime >= LastReferenceUpdate Then + ClearReference() LoadReferenceFromTheTextFile(FileName, LastProcessedObjectForDebug) End If From b1b4df428f66b5c2b0041b6c9782522121fd4217 Mon Sep 17 00:00:00 2001 From: Sergey Batanov Date: Sun, 19 Mar 2017 15:12:48 +0300 Subject: [PATCH 3/6] Elastic upload: convert data to UTC. --- EventLogLoaderService/EventLogProcessor.vb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index 18046ae..72b7d06 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -873,7 +873,7 @@ Public Class EventLogProcessor ESRecord.Severity = "Note" End Select - ESRecord.DateTime = EventRecord.DateTime + ESRecord.DateTime = EventRecord.DateTime.ToUniversalTime() ESRecord.ConnectID = EventRecord.ConnectID ESRecord.DataType = EventRecord.DataType ESRecord.SessionNumber = EventRecord.SessionNumber From cdcc9cb3c48a7fe0c848bbb2d8a3066c51377de1 Mon Sep 17 00:00:00 2001 From: Sergey Batanov Date: Sun, 19 Mar 2017 15:40:21 +0300 Subject: [PATCH 4/6] Fixed date filter in file log. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In case of file log and set date filter The Service analyze date in a filename and skips files with definitely unneeded data. В случае отбора по дате и журнала в файловом формате программа анализирует имя файла и отсекает те файлы, которые заведомо не содержат данных, подходящих под отбор по дате. --- EventLogLoaderService/EventLogProcessor.vb | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index 72b7d06..bf179a8 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -1200,7 +1200,45 @@ Public Class EventLogProcessor System.Array.Sort(ArrayFiles) + ' В случае с файловым журналом отсечём те файлы, которые заведомо не будут загружаться + + Dim ArrayFilesToProcess(0) As String + Dim FileCount = 0 + + System.Array.Reverse(ArrayFiles) For Each File In ArrayFiles + + Dim FI = My.Computer.FileSystem.GetFileInfo(File) + Dim DateTimePart = FI.Name.Substring(0, "yyyyMMddHHmmss".Length) + Dim Last = False + + Try + + Dim FileStartDateTime = Date.ParseExact(DateTimePart, "yyyyMMddHHmmss", CultureInfo.InvariantCulture) + If FileStartDateTime <= LoadEventsStartingAt + + ' Берём только один файл из тех, чья дата начала меньше даты отбора. + ' Все файлы младше заведомо не содержат данных, подходящих под отбор по дате + Last = True + + End If + + Catch + End Try + + ReDim Preserve ArrayFilesToProcess (FileCount) + ArrayFilesToProcess(FileCount) = File + FileCount += 1 + + If Last + Exit For + End If + + Next + + System.Array.Reverse(ArrayFilesToProcess) + + For Each File In ArrayFilesToProcess If Not File Is Nothing Then Try Dim FI = My.Computer.FileSystem.GetFileInfo(File) From a57eae6cfdfd955e019dfc12ea4587e82935d794 Mon Sep 17 00:00:00 2001 From: Sergey Batanov Date: Tue, 28 Mar 2017 16:42:54 +0300 Subject: [PATCH 5/6] Fixes positioning bug. When reading breaks in the middle of event, it will fail next iteration. Fix is to allocate on a line that seems to be a start of event. --- EventLogLoaderService/EventLogProcessor.vb | 81 ++++++++++++++++------ 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index bf179a8..89ff7c6 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -1395,25 +1395,32 @@ Public Class EventLogProcessor End Sub - Sub LoadEvents(FileName As String) + Function FindLineStartPosition(FS As FileStream, SourcePosition As Int64) As Int64 - Dim FS As FileStream = New FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite) - FS.Position = CurrentPosition + Dim Position = SourcePosition + While Position > 61 ' Magic: 61 - начальное положение - Dim SR As StreamReader = New StreamReader(FS) + Position = Position - 1 + FS.Position = Position + If FS.ReadByte() = 10 Then + + ' Нашли конецПредыдущей строки + Return FS.Position + End If + + End While + Return SourcePosition - 'Dim TextFile = My.Computer.FileSystem.OpenTextFileReader(FileName) + End Function + Sub LoadEvents(FileName As String) - 'TextFile.BaseStream.Position = Events.CurrentPosition ' учесть, что первые 2 символа служебные, т.е. первый - №3 - '' + 2 символа перевода каретки в конце каждой строки + Dim FS As FileStream = New FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite) + FS.Position = FindLineStartPosition(FS, CurrentPosition) - '' '' TEMP - ''Dim TextFile2 = My.Computer.FileSystem.OpenTextFileReader(FileName) - ''TextFile2.BaseStream.Position = Events.CurrentPosition + 1 - '' '' TEMP + Dim SR As StreamReader = New StreamReader(FS) Dim TextLine = "" @@ -1423,7 +1430,20 @@ Public Class EventLogProcessor Dim CountBracket = 0 Dim TextBlockOpen = False Dim Position = CurrentPosition - 'Dim WasReadSomeString = False + Dim StartLinePosition = CurrentPosition + Dim FirstLine = True + + + ' Костыль. + + ' Исходя из того, что предыдущее чтение могло остановитсья в любом положении, + ' идём до места, где строка похожа на начало события + + ' TODO: текущее событие оказывается не дочитано, потому (по идее) двигаться надо назад, а не вперёд + Dim DatePart = Path.GetFileNameWithoutExtension(FileName).Substring(0, 8) + Dim StartLinePattern = "{" + DatePart + + Dim LinesSkiped = 0 While Not TextLine Is Nothing @@ -1431,23 +1451,43 @@ Public Class EventLogProcessor TextLine = SR.ReadLine() If TextLine Is Nothing Then - 'если чтение прервано на середине, то ничего прибавлять не нужно, если же чтение - 'было завершено до конца, а потом читаются новые события, то появляется запятая, которую нужно учеть с ПЛЮС ОДИН позиции - 'If WasReadSomeString Then - ' Position = Position + 1 + If Not NewLine Then + + ' Если чтение прервано на середине события, нужно откатить позицию + CurrentPosition = StartLinePosition - ' Events.CurrentPosition = Position + End If - 'End If + Log.Debug("Last event text: {0}", StrEvent) Exit While End If - ' WasReadSomeString = True + If FirstLine Then + + If TextLine.StartsWith(StartLinePattern) Then + + FirstLine = False + If LinesSkiped > 0 Then + Log.Debug("Skipped {0} lines!", LinesSkiped) + End If + + Else + + If Not TextLine = "" Then + LinesSkiped += 1 + Log.Debug("Skiped: {0}", TextLine) + End If + + Continue While + + End If + + End If - Position = Position + 2 + Text.Encoding.UTF8.GetBytes(TextLine).Length + Position = FS.Position CurrentPosition = Position @@ -1459,6 +1499,7 @@ Public Class EventLogProcessor If ItsEndOfEvent(TextLine, CountBracket, TextBlockOpen) Then NewLine = True + StartLinePosition = CurrentPosition If Not StrEvent Is Nothing And (StrEvent.Length > 1) Then Try AddEvent(StrEvent) From 5835c5638ee24d46c3308a9dc075ec837b5601ea Mon Sep 17 00:00:00 2001 From: Sergey Batanov Date: Thu, 5 Oct 2017 12:03:21 +0300 Subject: [PATCH 6/6] Fixed sqlite->ElasticSearch couple. --- EventLogLoaderService/EventLogProcessor.vb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index 1fece12..67ac81b 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -876,7 +876,7 @@ Public Class EventLogProcessor ESRecord.Severity = "Note" End Select - ESRecord.DateTime = EventRecord.DateTime.ToUniversalTime() + ESRecord.DateTime = EventRecord.DateTime ESRecord.ConnectID = EventRecord.ConnectID ESRecord.DataType = EventRecord.DataType ESRecord.SessionNumber = EventRecord.SessionNumber @@ -1336,7 +1336,7 @@ Public Class EventLogProcessor OneEvent.Severity = rs("severity") OneEvent.ConnectID = rs("connectID") - OneEvent.DateTime = New Date().AddSeconds(Convert.ToInt64(rs("date") / 10000)) + OneEvent.DateTime = New Date().AddSeconds(Convert.ToInt64(rs("date") / 10000)) ' date-time in UTC OneEvent.TransactionStatus = rs("transactionStatus") OneEvent.TransactionMark = rs("transactionID") @@ -1630,7 +1630,8 @@ Public Class EventLogProcessor Dim OneEvent As OneEventRecord = New OneEventRecord Dim Array = ParserServices.ParseEventLogString(Str) - OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider) + OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider) ' local date-time + OneEvent.DateTime = OneEvent.DateTime.ToUniversalTime OneEvent.TransactionStatus = Array(1) If OneEvent.DateTime < LoadEventsStartingAt Then @@ -1648,7 +1649,7 @@ Public Class EventLogProcessor End If Catch ex As Exception End Try - + OneEvent.TransactionStartTime = OneEvent.TransactionStartTime.ToUniversalTime OneEvent.TransactionMark = From16To10(TransStr.Substring(TransStr.IndexOf(",") + 1)) OneEvent.Transaction = Array(2)