diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index c0d83d6..67ac81b 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -771,7 +771,15 @@ Public Class EventLogProcessor copy.ColumnMappings.Add(jj, jj) Next copy.DestinationTableName = "Events" - copy.WriteToServer(dt) + Try + copy.WriteToServer(dt) + Catch ex As InvalidOperationException + Log.Error("Ошибка сохранения в БД записи по ИБ " + InfobaseName + " : " + ex.Message) + Catch ex As Exception + Log.Error("Ошибка сохранения в БД записи по ИБ " + InfobaseName + " : " + ex.Message) + End Try + 'InvalidOperationException + End Using SaveReadParametersToFile() @@ -998,7 +1006,7 @@ Public Class EventLogProcessor LastProcessedObjectForDebug = TextObject - Dim a = ParserServices.ParseEventLogString(TextObject) + Dim a = ParserServices.ParseEventLogString(TextObject.ToString().Trim()) If Not a Is Nothing Then Select Case a(0) @@ -1038,7 +1046,7 @@ Public Class EventLogProcessor End Sub - Sub LoadReference() + Sub ClearReference() 'Clear all reference dictionaries DictUsers.Clear() @@ -1050,10 +1058,16 @@ Public Class EventLogProcessor DictMainPorts.Clear() DictSecondPorts.Clear() + End Sub + + Sub LoadReference() + Dim FileName = Path.Combine(Catalog, "1Cv8.lgd") If My.Computer.FileSystem.FileExists(FileName) Then + ClearReference() + Try Dim Conn = New SQLite.SQLiteConnection("Data Source=" + FileName) Conn.Open() @@ -1138,6 +1152,7 @@ Public Class EventLogProcessor If FI.LastWriteTime >= LastReferenceUpdate Then + ClearReference() LoadReferenceFromTheTextFile(FileName, LastProcessedObjectForDebug) End If @@ -1202,7 +1217,45 @@ Public Class EventLogProcessor System.Array.Sort(ArrayFiles) + ' В случае с файловым журналом отсечём те файлы, которые заведомо не будут загружаться + + Dim ArrayFilesToProcess(0) As String + Dim FileCount = 0 + + System.Array.Reverse(ArrayFiles) For Each File In ArrayFiles + + Dim FI = My.Computer.FileSystem.GetFileInfo(File) + Dim DateTimePart = FI.Name.Substring(0, "yyyyMMddHHmmss".Length) + Dim Last = False + + Try + + Dim FileStartDateTime = Date.ParseExact(DateTimePart, "yyyyMMddHHmmss", CultureInfo.InvariantCulture) + If FileStartDateTime <= LoadEventsStartingAt + + ' Берём только один файл из тех, чья дата начала меньше даты отбора. + ' Все файлы младше заведомо не содержат данных, подходящих под отбор по дате + Last = True + + End If + + Catch + End Try + + ReDim Preserve ArrayFilesToProcess (FileCount) + ArrayFilesToProcess(FileCount) = File + FileCount += 1 + + If Last + Exit For + End If + + Next + + System.Array.Reverse(ArrayFilesToProcess) + + For Each File In ArrayFilesToProcess If Not File Is Nothing Then Try Dim FI = My.Computer.FileSystem.GetFileInfo(File) @@ -1283,7 +1336,7 @@ Public Class EventLogProcessor OneEvent.Severity = rs("severity") OneEvent.ConnectID = rs("connectID") - OneEvent.DateTime = New Date().AddSeconds(Convert.ToInt64(rs("date") / 10000)) + OneEvent.DateTime = New Date().AddSeconds(Convert.ToInt64(rs("date") / 10000)) ' date-time in UTC OneEvent.TransactionStatus = rs("transactionStatus") OneEvent.TransactionMark = rs("transactionID") @@ -1361,25 +1414,32 @@ Public Class EventLogProcessor End Sub - Sub LoadEvents(FileName As String) + Function FindLineStartPosition(FS As FileStream, SourcePosition As Int64) As Int64 - Dim FS As FileStream = New FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite) - FS.Position = CurrentPosition + Dim Position = SourcePosition + While Position > 61 ' Magic: 61 - начальное положение - Dim SR As StreamReader = New StreamReader(FS) + Position = Position - 1 + FS.Position = Position + If FS.ReadByte() = 10 Then + ' Нашли конецПредыдущей строки + Return FS.Position + End If - 'Dim TextFile = My.Computer.FileSystem.OpenTextFileReader(FileName) + End While + + Return SourcePosition + End Function - 'TextFile.BaseStream.Position = Events.CurrentPosition ' учесть, что первые 2 символа служебные, т.е. первый - №3 - '' + 2 символа перевода каретки в конце каждой строки + Sub LoadEvents(FileName As String) - '' '' TEMP - ''Dim TextFile2 = My.Computer.FileSystem.OpenTextFileReader(FileName) - ''TextFile2.BaseStream.Position = Events.CurrentPosition + 1 - '' '' TEMP + Dim FS As FileStream = New FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite) + FS.Position = FindLineStartPosition(FS, CurrentPosition) + + Dim SR As StreamReader = New StreamReader(FS) Dim TextLine = "" @@ -1389,7 +1449,20 @@ Public Class EventLogProcessor Dim CountBracket = 0 Dim TextBlockOpen = False Dim Position = CurrentPosition - 'Dim WasReadSomeString = False + Dim StartLinePosition = CurrentPosition + Dim FirstLine = True + + + ' Костыль. + + ' Исходя из того, что предыдущее чтение могло остановитсья в любом положении, + ' идём до места, где строка похожа на начало события + + ' TODO: текущее событие оказывается не дочитано, потому (по идее) двигаться надо назад, а не вперёд + Dim DatePart = Path.GetFileNameWithoutExtension(FileName).Substring(0, 8) + Dim StartLinePattern = "{" + DatePart + + Dim LinesSkiped = 0 While Not TextLine Is Nothing @@ -1397,23 +1470,43 @@ Public Class EventLogProcessor TextLine = SR.ReadLine() If TextLine Is Nothing Then - 'если чтение прервано на середине, то ничего прибавлять не нужно, если же чтение - 'было завершено до конца, а потом читаются новые события, то появляется запятая, которую нужно учеть с ПЛЮС ОДИН позиции - 'If WasReadSomeString Then - ' Position = Position + 1 + If Not NewLine Then + + ' Если чтение прервано на середине события, нужно откатить позицию + CurrentPosition = StartLinePosition - ' Events.CurrentPosition = Position + End If - 'End If + Log.Debug("Last event text: {0}", StrEvent) Exit While End If - ' WasReadSomeString = True + If FirstLine Then + + If TextLine.StartsWith(StartLinePattern) Then + + FirstLine = False + If LinesSkiped > 0 Then + Log.Debug("Skipped {0} lines!", LinesSkiped) + End If + + Else + + If Not TextLine = "" Then + LinesSkiped += 1 + Log.Debug("Skiped: {0}", TextLine) + End If - Position = Position + 2 + Text.Encoding.UTF8.GetBytes(TextLine).Length + Continue While + + End If + + End If + + Position = FS.Position CurrentPosition = Position @@ -1425,11 +1518,12 @@ Public Class EventLogProcessor If ItsEndOfEvent(TextLine, CountBracket, TextBlockOpen) Then NewLine = True - If Not StrEvent Is Nothing Then + StartLinePosition = CurrentPosition + If Not StrEvent Is Nothing And (StrEvent.Length > 1) Then Try AddEvent(StrEvent) Catch ex As Exception - + Log.Error(ex.Message) End Try '*** @@ -1466,7 +1560,7 @@ Public Class EventLogProcessor End If Next - ItsEndOfEvent = (Count = 0) + ItsEndOfEvent = (Count = 0) And Not TextBlockOpen End Function @@ -1536,7 +1630,8 @@ Public Class EventLogProcessor Dim OneEvent As OneEventRecord = New OneEventRecord Dim Array = ParserServices.ParseEventLogString(Str) - OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider) + OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider) ' local date-time + OneEvent.DateTime = OneEvent.DateTime.ToUniversalTime OneEvent.TransactionStatus = Array(1) If OneEvent.DateTime < LoadEventsStartingAt Then @@ -1554,7 +1649,7 @@ Public Class EventLogProcessor End If Catch ex As Exception End Try - + OneEvent.TransactionStartTime = OneEvent.TransactionStartTime.ToUniversalTime OneEvent.TransactionMark = From16To10(TransStr.Substring(TransStr.IndexOf(",") + 1)) OneEvent.Transaction = Array(2) diff --git a/EventLogLoaderService/Parser.vb b/EventLogLoaderService/Parser.vb index f3c1959..47ceb45 100644 --- a/EventLogLoaderService/Parser.vb +++ b/EventLogLoaderService/Parser.vb @@ -4,36 +4,53 @@ Dim ArrayLines(0) - Dim Text2 = Text.Substring(1, IIf(Text.EndsWith(","), Text.Length - 3, Text.Length - 2)) + "," + If Text Is Nothing Or Text.Length = 0 Then + Return ArrayLines + End If - Dim Str = "" + ' Сим отсекаем скобочки + Dim Text2 = Text.Substring(1, IIf(Text.EndsWith(","), Text.Length - 3, Text.Length - 2)) + "," - Dim Delim = Text2.IndexOf(",") Dim i = 0 + Dim Str = "" + Dim TextBlockOpen = False + Dim count = 0 + Dim BracketCount = 0 - While Delim > 0 - Str = Str + Text2.Substring(0, Delim).Trim - Text2 = Text2.Substring(Delim + 1) + For i = 0 To Text2.Length - 1 - If CountSubstringInString(Str, "{") = CountSubstringInString(Str, "}") _ - And Math.IEEERemainder(CountSubstringInString(Str, """"), 2) = 0 Then + Dim simb = Text2.Substring(i, 1) - ReDim Preserve ArrayLines(i) + If (simb = ",") And Not TextBlockOpen And (BracketCount = 0) Then - If Str.StartsWith("""") And Str.EndsWith("""") Then - Str = Str.Substring(1, Str.Length - 2) - End If + ReDim Preserve ArrayLines(count) + ArrayLines(count) = Str + count = count + 1 - ArrayLines(i) = Str - i = i + 1 Str = "" + + Else If simb = "{" And Not TextBlockOpen Then + + BracketCount = BracketCount + 1 + Str = Str + simb + + Else If simb = "}" And Not TextBlockOpen Then + + BracketCount = BracketCount - 1 + Str = Str + simb + + Else If simb = """" Then + + TextBlockOpen = Not TextBlockOpen + Str = Str + """" + Else - Str = Str + "," - End If - Delim = Text2.IndexOf(",") + Str = Str + simb + + End If - End While + Next Return ArrayLines