diff --git a/src/main/java/emissary/util/magic/MagicNumber.java b/src/main/java/emissary/util/magic/MagicNumber.java index 0606b13faa..683d49ee26 100644 --- a/src/main/java/emissary/util/magic/MagicNumber.java +++ b/src/main/java/emissary/util/magic/MagicNumber.java @@ -8,9 +8,10 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Deque; +import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.Queue; public class MagicNumber { @@ -144,7 +145,9 @@ private static String escapeBackspace(String desc) { StringBuilder s = new StringBuilder(); for (int i = 0; i < desc.length(); i++) { if (desc.charAt(i) == '\\' && (i + 1) < desc.length() && desc.charAt(i + 1) == 'b') { - s = new StringBuilder(s.substring(0, s.length() - 1)); + if (s.length() > 0) { + s.setLength(s.length() - 1); + } i++; continue; } @@ -172,16 +175,16 @@ private String format(String desc, byte[] data) { if (!substitute) { return desc; } - Deque chars = new ArrayDeque<>(); - for (int i = desc.length() - 1; i >= 0; --i) { - chars.push(desc.charAt(i)); + Queue chars = new ArrayDeque<>(); + for (int i = 0; i < desc.length(); i++) { + chars.add(desc.charAt(i)); } StringBuilder sb = new StringBuilder(); while (!chars.isEmpty()) { - Character next = chars.pop(); + Character next = chars.poll(); if (!chars.isEmpty() && next == '%') { - char subType = chars.pop(); + char subType = chars.poll(); if (dataType == TYPE_STRING) { if (offset < (data.length - 2)) { String sub = new String(Objects.requireNonNull(getElement(data, offset, 1)), DEFAULT_CHARSET); @@ -205,11 +208,11 @@ private String format(String desc, byte[] data) { } if (subType == 'l' && !chars.isEmpty() && chars.peek() == 'd') { - chars.pop(); + chars.poll(); } continue; } - sb.append(next.charValue()); + sb.append(next); } return sb.toString(); } @@ -263,11 +266,29 @@ private boolean testNumeric(byte[] data) { if (substitute) { return true; } + + if (mask != null && mask.length == data.length) { + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (data[i] & mask[i]); + } + } + + // Short-circuit instantly for strings + if (dataType == TYPE_STRING) { + return Arrays.equals(data, value); + } + byte[] mValues = value; + if (mValues == null || data.length != mValues.length) { + return false; + } + + int end = mValues.length; + boolean isBigEndian = dataType == TYPE_BESHORT || dataType == TYPE_BELONG || dataType == TYPE_BEDATE || + dataType == TYPE_SHORT || dataType == TYPE_LONG || dataType == TYPE_BYTE; log.debug("Unary Operator: {}", unaryOperator); - int end = mValues.length; switch (unaryOperator) { case MAGICOPERATOR_AND: case MAGICOPERATOR_BWAND: @@ -277,59 +298,63 @@ private boolean testNumeric(byte[] data) { } } return true; - case MAGICOPERATOR_GTHAN: - for (int i = 0; i < end; i++) { - if ((data[i] & 0xFF) < (mValues[i] & 0xFF)) { - return false; - } - if (i == end - 1 && data[i] == mValues[i]) { - return false; - } - } - return true; - case MAGICOPERATOR_LTHAN: - for (int i = 0; i < end; i++) { - if ((data[i] & 0xFF) > (mValues[i] & 0xFF)) { - return false; - } - if (i == end - 1 && data[i] == mValues[i]) { - return false; - } - } - return true; case MAGICOPERATOR_OR: for (int i = 0; i < end; i++) { - if (data[i] == mValues[i]) { + if ((data[i] & mValues[i]) != 0) { return true; } } return false; - case MAGICOPERATOR_BWNOT: case MAGICOPERATOR_NOT: + case MAGICOPERATOR_BWNOT: for (int i = 0; i < end; i++) { if (data[i] != mValues[i]) { return true; } } return false; + case MAGICOPERATOR_GTHAN: case MAGICOPERATOR_EQUAL_GTHAN: - for (int i = 0; i < end; i++) { - if ((data[i] & 0xFF) < (mValues[i] & 0xFF)) { - return false; + case MAGICOPERATOR_LTHAN: + case MAGICOPERATOR_EQUAL_LTHAN: + int cmp = 0; + if (isBigEndian) { + // Big Endian: MSB is at index 0 + for (int i = 0; i < end; i++) { + int v1 = data[i] & 0xFF; + int v2 = mValues[i] & 0xFF; + if (v1 != v2) { + cmp = Integer.compare(v1, v2); + break; + } + } + } else { + // Little Endian: MSB is at the last index + for (int i = end - 1; i >= 0; i--) { + int v1 = data[i] & 0xFF; + int v2 = mValues[i] & 0xFF; + if (v1 != v2) { + cmp = Integer.compare(v1, v2); + break; + } } } - return true; - case MAGICOPERATOR_EQUAL_LTHAN: - for (int i = 0; i < end; i++) { - if ((data[i] & 0xFF) > (mValues[i] & 0xFF)) { + + switch (unaryOperator) { + case MAGICOPERATOR_GTHAN: + return cmp > 0; + case MAGICOPERATOR_EQUAL_GTHAN: + return cmp >= 0; + case MAGICOPERATOR_LTHAN: + return cmp < 0; + case MAGICOPERATOR_EQUAL_LTHAN: + return cmp <= 0; + default: return false; - } } - return true; default: throw new IllegalStateException( "This MagicNumber instance is configured incorrectly. The unary operator is set to an unknown or unconfigured value."); - } } @@ -397,7 +422,12 @@ public String toString() { } else { sb.append(unaryOperator); } - sb.append(MagicMath.byteArrayToHexString(value)); + + if (dataType == TYPE_STRING && value != null) { + sb.append(new String(value, DEFAULT_CHARSET)); + } else { + sb.append(MagicMath.byteArrayToHexString(value)); + } sb.append('\t'); sb.append(description); diff --git a/src/main/java/emissary/util/magic/MagicNumberFactory.java b/src/main/java/emissary/util/magic/MagicNumberFactory.java index 56b2bbf66d..843fb9399c 100644 --- a/src/main/java/emissary/util/magic/MagicNumberFactory.java +++ b/src/main/java/emissary/util/magic/MagicNumberFactory.java @@ -84,11 +84,9 @@ public static List buildMagicNumberList(byte[] configData, @Nullabl try { if (depth == 0 && !extensions.isEmpty()) { if (finger == null) { - extensions = null; extensions = new ArrayList<>(); } else { addExtensionsLayer(extensions, finger); - extensions = null; extensions = new ArrayList<>(); finger = null; } @@ -119,7 +117,6 @@ public static List buildMagicNumberList(byte[] configData, @Nullabl } currentDepth = depth; addExtensionsLayer(extensions, finger); - extensions = null; extensions = new ArrayList<>(); parseAndStore(extensions, s, swallowParseException); } @@ -140,11 +137,7 @@ public static List buildMagicNumberList(byte[] configData, @Nullabl if (depth > 0) { MagicNumber mItem = magicNumberList.get(magicNumberList.size() - 1); String signature = mItem.toString(); - List failedExtensions = continuationErrorMap.get(signature); - if (failedExtensions == null) { - failedExtensions = new ArrayList<>(); - continuationErrorMap.put(mItem.toString(), failedExtensions); - } + List failedExtensions = continuationErrorMap.computeIfAbsent(signature, k -> new ArrayList<>()); failedExtensions.add("[MAGIC LINE# " + counter + "] " + s); } else { // depth = 0 @@ -190,11 +183,7 @@ private static MagicNumber parseAndStore(List storage, String entry * @param extensions a {@link List} of continuations which are MagicNumber instances */ private static void addExtensionsLayer(List extensions, MagicNumber target) { - MagicNumber[] extensionArray = new MagicNumber[extensions.size()]; - int index = 0; - for (MagicNumber m : extensions) { - extensionArray[index++] = m; - } + MagicNumber[] extensionArray = extensions.toArray(new MagicNumber[0]); target.addDependencyLayer(extensionArray); } @@ -246,7 +235,10 @@ public static MagicNumber buildMagicNumber(String entry, boolean swallowParseExc // column C parsing item.unaryOperator = resolveUnary(columns, item); item.value = resolveValue(columns, item); - item.dataTypeLength = item.value.length; + + if (item.dataType == MagicNumber.TYPE_STRING && item.value != null) { + item.dataTypeLength = item.value.length; + } } catch (Exception e) { throw new ParseException("Error on column 2:" + columns[2] + ". " + e.getMessage()); } @@ -261,10 +253,8 @@ public static MagicNumber buildMagicNumber(String entry, boolean swallowParseExc private static String[] tokenizeEntry(String entry) { int index = 0; String[] columns = new String[4]; - columns[0] = EMPTYSTRING; - columns[1] = EMPTYSTRING; - columns[2] = EMPTYSTRING; - columns[3] = EMPTYSTRING; + Arrays.fill(columns, EMPTYSTRING); + for (int i = 0; i < entry.length(); i++) { char c = entry.charAt(i); if (c == '\\' && i != (entry.length() - 1) && entry.charAt(i + 1) == ' ') { @@ -308,10 +298,7 @@ private static String[] prepareEntry(String entry) throws ParseException { String[] columns = tokenizeEntry(subject); for (int count = 0; count < columns.length; count++) { - if (count == 3 && columns[count].isEmpty() && columns[0].charAt(0) != '>') { - // columns[count] = NULL_DESCRIPTION; - - } else if (columns[count].isEmpty() && count < 3) { + if (columns[count].isEmpty() && count < 3) { throw new ParseException(ENTRY_4COLUMN_RULE); } } @@ -339,7 +326,7 @@ private static int resolveOffset(String[] columns, MagicNumber item) throws Pars } private static char resolveOffsetUnary(String[] columns) { - if (columns[0].charAt(0) == '&') { + if (!columns[0].isEmpty() && columns[0].charAt(0) == '&') { return '&'; } return (char) 0; @@ -388,7 +375,10 @@ private static int resolveDataType(String[] columns) throws ParseException { } private static int lookupDataType(String arg) { - int dataTypeIdInt = typeMap.get(arg.toUpperCase(Locale.getDefault())); + Integer dataTypeIdInt = typeMap.get(arg.toUpperCase(Locale.getDefault())); + if (dataTypeIdInt == null) { + return -1; + } switch (dataTypeIdInt) { case MagicNumber.TYPE_DATE: case MagicNumber.TYPE_BEDATE: @@ -404,7 +394,7 @@ private static byte[] resolveMask(String[] columns, MagicNumber item) { int ix = columns[1].indexOf("&"); if (ix > 0) { byte[] maskValues = MagicMath.stringToByteArray(columns[1].substring(ix + 1)); - MagicMath.setLength(maskValues, item.dataTypeLength); + return MagicMath.setLength(maskValues, item.dataTypeLength); } return null; } @@ -416,9 +406,7 @@ private static byte[] resolveValue(String[] columns, MagicNumber item) { String subject = columns[2]; if (item.dataType == MagicNumber.TYPE_STRING && !(subject.length() == 1 && subject.charAt(0) == 'x')) { - byte[] strVal = MagicMath.parseEscapedString(subject); - item.dataTypeLength = strVal.length; - return strVal; + return MagicMath.parseEscapedString(subject); } else if (subject.length() == 1 && subject.charAt(0) == 'x') { item.substitute = true; return new byte[0]; @@ -434,9 +422,6 @@ private static byte[] resolveValue(String[] columns, MagicNumber item) { byte[] valueArray = MagicMath.stringToByteArray(subject); valueArray = MagicMath.setLength(valueArray, item.dataTypeLength); - if (item.mask != null) { - valueArray = MagicMath.mask(valueArray, item.mask); - } if (item.dataType == MagicNumber.TYPE_LELONG) { MagicMath.longEndianSwap(valueArray, 0); } else if (item.dataType == MagicNumber.TYPE_LESHORT) { @@ -462,7 +447,6 @@ private static int unaryPrefixLength(@Nullable String s) { case MagicNumber.MAGICOPERATOR_NOT: return 1; case MagicNumber.MAGICOPERATOR_GTHAN: - return len > 1 && s.charAt(1) == MagicNumber.MAGICOPERATOR_AND ? 2 : 1; case MagicNumber.MAGICOPERATOR_LTHAN: return len > 1 && s.charAt(1) == MagicNumber.MAGICOPERATOR_AND ? 2 : 1; default: @@ -476,11 +460,7 @@ private static int getDataTypeByteLength(MagicNumber item) { int dataTypeId = item.dataType; switch (dataTypeId) { case MagicNumber.TYPE_STRING: - if (item.value == null) { - return -1; - } else { - return item.value.length; - } + return (item.value == null) ? -1 : item.value.length; case MagicNumber.TYPE_BYTE: return 1; case MagicNumber.TYPE_SHORT: diff --git a/src/test/java/emissary/util/magic/MagicNumberTest.java b/src/test/java/emissary/util/magic/MagicNumberTest.java index 6b574e38c3..cdd46424b7 100644 --- a/src/test/java/emissary/util/magic/MagicNumberTest.java +++ b/src/test/java/emissary/util/magic/MagicNumberTest.java @@ -3,6 +3,8 @@ import emissary.test.core.junit5.UnitTest; import jakarta.xml.bind.DatatypeConverter; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -84,12 +86,17 @@ void testLessThanBelong() throws ParseException { } @Test - void testLessEqualBelong() throws ParseException { + void testLessEqualBelong() throws ParseException, DecoderException { // ABCD MagicNumber m = MagicNumberFactory.buildMagicNumber("0 belong <=0x41424344 FOO"); assertTrue(m.test("ABCC".getBytes()), "LessEqual magic operator failed"); assertTrue(m.test("ABCD".getBytes()), "LessEqual magic operator failed"); assertFalse(m.test("ABCE".getBytes()), "LessEqual magic operator failed"); + + m = MagicNumberFactory.buildMagicNumber("0 belong <=0x00010000 FOO"); + assertTrue(m.test(Hex.decodeHex("00010000")), "Big Endian less than equal failed on exact match"); + assertTrue(m.test(Hex.decodeHex("0000FFFF")), "Big Endian less than equal failed"); + assertFalse(m.test(Hex.decodeHex("00020000")), "Big Endian less than equal failed"); } @Test @@ -101,12 +108,23 @@ void testBeshort() throws ParseException { } @Test - void testGreaterThanBeshort() throws ParseException { + void testGreaterThanBeshort() throws ParseException, DecoderException { // AB MagicNumber m = MagicNumberFactory.buildMagicNumber("0 beshort >0x4142 FOO"); assertTrue(m.test("AC".getBytes()), "Greater than magic operator failed"); assertFalse(m.test("AB".getBytes()), "Greater than magic operator failed"); assertFalse(m.test("AA".getBytes()), "Greater than magic operator failed"); + + m = MagicNumberFactory.buildMagicNumber("0 beshort >0x0000 FOO"); + assertFalse(m.test(Hex.decodeHex("0000")), "Greater than magic operator failed"); + assertTrue(m.test(Hex.decodeHex("0001")), "Greater than magic operator failed"); + assertTrue(m.test(Hex.decodeHex("0101")), "Greater than magic operator failed"); + assertTrue(m.test(Hex.decodeHex("0100")), "Greater than magic operator failed"); + + m = MagicNumberFactory.buildMagicNumber("0 beshort >0x0100 FOO"); + assertTrue(m.test(Hex.decodeHex("0200")), "Big Endian greater than failed on MSB"); + assertFalse(m.test(Hex.decodeHex("00FF")), "Big Endian greater than failed on MSB tie-breaker"); + assertTrue(m.test(Hex.decodeHex("0101")), "Big Endian greater than failed on LSB tie-breaker"); } @Test @@ -136,6 +154,48 @@ void testLessEqualBeshort() throws ParseException { assertFalse(m.test("AC".getBytes()), "LessEqual magic operator failed"); } + @Test + void testGreaterThanLeshort() throws ParseException, DecoderException { + MagicNumber m = MagicNumberFactory.buildMagicNumber("0 leshort >0x0001 FOO"); + assertTrue(m.test(Hex.decodeHex("0200")), "Little Endian greater than failed on LSB increment"); + assertTrue(m.test(Hex.decodeHex("0001")), "Little Endian greater than failed on MSB increment"); + assertFalse(m.test(Hex.decodeHex("0000")), "Little Endian greater than failed on lower value"); + } + + @Test + void testLessThanEqualLelong() throws ParseException, DecoderException { + MagicNumber m = MagicNumberFactory.buildMagicNumber("0 lelong <=0x00000100 FOO"); + assertTrue(m.test(Hex.decodeHex("00010000")), "Little Endian less than equal failed on exact match"); + assertTrue(m.test(Hex.decodeHex("FF000000")), "Little Endian less than equal failed on lower value"); + assertFalse(m.test(Hex.decodeHex("01010000")), "Little Endian less than equal failed on higher value"); + } + + @Test + void testSameInputBigEndianVsLittleEndian() throws ParseException, DecoderException { + MagicNumber bem = MagicNumberFactory.buildMagicNumber("0 beshort >0x0005 FOO"); + assertTrue(bem.test(Hex.decodeHex("0200")), "Big Endian failed: 0x0200 (512) should be greater than 5"); + + MagicNumber lem = MagicNumberFactory.buildMagicNumber("0 leshort >0x0005 FOO"); + assertFalse(lem.test(Hex.decodeHex("0200")), "Little Endian failed: 0x0200 (2) should NOT be greater than 5"); + } + + @Test + void testOrBeshort() throws ParseException, DecoderException { + MagicNumber m = MagicNumberFactory.buildMagicNumber("0 beshort x0x1010 FOO"); + + assertFalse(m.test(Hex.decodeHex("0000")), "OR magic operator failed"); + assertFalse(m.test(Hex.decodeHex("0001")), "OR magic operator failed"); + assertFalse(m.test(Hex.decodeHex("0101")), "OR magic operator failed"); + + assertTrue(m.test(Hex.decodeHex("0010")), "OR magic operator failed"); + assertTrue(m.test(Hex.decodeHex("0011")), "OR magic operator failed"); + assertTrue(m.test(Hex.decodeHex("1000")), "OR magic operator failed"); + assertTrue(m.test(Hex.decodeHex("1010")), "OR magic operator failed"); + assertTrue(m.test(Hex.decodeHex("1100")), "OR magic operator failed"); + assertTrue(m.test(Hex.decodeHex("1111")), "OR magic operator failed"); + assertTrue(m.test(Hex.decodeHex("FFFF")), "OR magic operator failed"); + } + @Test void testString() throws ParseException { // ABCD