Save current state with some ROST extra features.

4 files changed, 941 insertions, 3 deletions

diff --git a/tests/analysis/scan/functions.py b/tests/analysis/scan/functions.py
index 8553018..96f029f 100644
--- a/tests/analysis/scan/functions.py
+++ b/tests/analysis/scan/functions.py
@@ -76,6 +76,64 @@ rule test {
             self.check_rule_success(rule, cnt)
 
 
+    def testMathOperations(self):
+        """Perform math operations with core functions."""
+
+        rule = '''
+rule test {
+
+   condition:
+      math.to_string(123) == "123"
+         and math.to_string(291, 16) == "0x123"
+         and math.to_string(-83, 8) == "-0123"
+         and math.to_string(123, 2) == "0b1111011"
+
+}
+'''
+
+        self.check_rule_success(rule)
+
+
+    def testStringOperations(self):
+        """Perform string operations with core functions."""
+
+        rule = '''
+rule test {
+
+   condition:
+      string.lower("ABCd") == "abcd" and string.lower("123abc") == "123abc"
+
+}
+'''
+
+        self.check_rule_success(rule)
+
+        rule = '''
+rule test {
+
+   condition:
+      string.upper("abcD") == "ABCD" and string.upper("123ABC") == "123ABC"
+
+}
+'''
+
+        self.check_rule_success(rule)
+
+        rule = '''
+rule test {
+
+   condition:
+      string.to_int("123") == 123
+         and string.to_int("123", 16) == 291
+         and string.to_int("0x123") == 291
+         and string.to_int("-0123") == -83
+
+}
+'''
+
+        self.check_rule_success(rule)
+
+
     def testTime(self):
         """Check current time."""
 
diff --git a/tests/analysis/scan/pyapi.py b/tests/analysis/scan/pyapi.py
index b5b2453..4b066f3 100644
--- a/tests/analysis/scan/pyapi.py
+++ b/tests/analysis/scan/pyapi.py
@@ -6,7 +6,6 @@ from gi._constants import TYPE_INVALID
 from pychrysalide.analysis.scan import ScanExpression
 from pychrysalide.analysis.scan import ScanOptions
 from pychrysalide.analysis.scan import find_token_modifiers_for_name
-from pychrysalide.analysis.scan.patterns.modifiers import PlainModifier
 from pychrysalide.glibext import ComparableItem
 
 
diff --git a/tests/analysis/scan/scanning_hex.py b/tests/analysis/scan/scanning_hex.py
index e009b79..32979c8 100644
--- a/tests/analysis/scan/scanning_hex.py
+++ b/tests/analysis/scan/scanning_hex.py
@@ -3,8 +3,218 @@ from common import RostTestClass
 from pychrysalide.analysis.contents import MemoryContent
 
 
-class TestRostScanning(RostTestClass):
-    """TestCases for the bytes section syntax."""
+class TestRostScanningBinary(RostTestClass):
+    """TestCases for the bytes section syntax (binary)."""
+
+    def testLonelyPatterns(self):
+        """Evaluate the most simple patterns."""
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 41 }
+
+   condition:
+      #a == 1 and @a[0] == 0
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 62 }
+
+   condition:
+      #a == 1 and @a[0] == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 66 }
+
+   condition:
+      #a == 1 and @a[0] == 5
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?1 }
+
+   condition:
+      #a == 1 and @a[0] == 0
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?2 }
+
+   condition:
+      #a == 1 and @a[0] == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?6 }
+
+   condition:
+      #a == 1 and @a[0] == 5
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testLonelyPatternsNot(self):
+        """Evaluate the most simple patterns (not version)."""
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ~41 }
+
+   condition:
+      #a == 5 and @a[0] == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ~62 }
+
+   condition:
+      #a == 5 and @a[0] == 0
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ~66 }
+
+   condition:
+      #a == 5 and @a[4] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ~?1 }
+
+   condition:
+      #a == 5 and @a[0] == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ~?2 }
+
+   condition:
+      #a == 5 and @a[0] == 0
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'Abcdef')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ~?6 }
+
+   condition:
+      #a == 5 and @a[4] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
 
     def testSimpleHexPattern(self):
         """Test a simple hex pattern."""
@@ -24,3 +234,480 @@ rule test {
 '''
 
         self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 2d 41 62 63 }
+
+   condition:
+      #a == 1 and @a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testSimpleMaskedHexPattern(self):
+        """Test a simple masked hex pattern."""
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?1 6? ?3 }
+
+   condition:
+      #a == 1 and @a[0] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testHexPatternWithPlainAndMasked(self):
+        """Test hex patterns with plain and masked bytes."""
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 41 6? ?3 }
+
+   condition:
+      #a == 1 and @a[0] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 4? 62 ?3 }
+
+   condition:
+      #a == 1 and @a[0] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 4? ?2 63 }
+
+   condition:
+      #a == 1 and @a[0] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 4? ?2 ?3 }
+
+   condition:
+      #a == 1 and @a[0] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 2d 4? ?2 63 }
+
+   condition:
+      #a == 1 and @a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 2d 4? 62 ?3 2d }
+
+   condition:
+      #a == 1 and @a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 2? 41 6? 63 ?d }
+
+   condition:
+      #a == 1 and @a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testHexPatternWithPlainAndHoles(self):
+        """Test hex patterns with plain bytes and holes."""
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 33 ?? 41 ?? 63 ?? 34 }
+
+   condition:
+      #a == 1 and @a[0] == 2
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?? 33 ?? 41 ?? 63 ?? 34 ?? }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?? 33 [1-5] 63 ?? 34 ?? }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { [3-4] 41 ?? 63 ?? 34 ?? }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?? 33 ?? 41 ?? 63 [3-] }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testHexPatternWithMaskedAndHoles(self):
+        """Test hex patterns with masked bytes and holes."""
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?3 ?? 4? ?? 6? ?? ?4 }
+
+   condition:
+      #a == 1 and @a[0] == 2
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?? ?3 ?? 4? ?? 6? ?? ?4 ?? }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?? ?3 [1-5] ?3 ?? ?4 ?? }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { [3-4] ?1 ?? ?3 ?? ?4 ?? }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ?? 3? ?? 4? ?? 6? [3-] }
+
+   condition:
+      #a == 1 and @a[0] == 1 and !a[0] == 9
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testPipedPlainHexPatterns(self):
+        """Look for several patterns at once with piped definition."""
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 41 62 ( 63 | 64 | 65 ) }
+
+   condition:
+      #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ( 41 | f2 | f3 ) 62 63 }
+
+   condition:
+      #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 41 ( 61 | 62 | 63 ) 63 }
+
+   condition:
+      #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ( 41 62 63 | 42 62 63 | 43 62 63 ) }
+
+   condition:
+      #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testPipedMaskedHexPatterns(self):
+        """Look for several patterns at once with piped definition."""
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 4? 6? ( ?3 | ?4 | ?5 ) }
+
+   condition:
+      #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ( ?1 | ?2 | ?3 ) 6? 6? }
+
+   condition:
+      console.log("COUNTER: ", #a) and #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { 4? ( ?1 | ?2 | ?3 ) 6? }
+
+   condition:
+      #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = { ( 4? ?2 ?3 | 4? 6? 6? | ?3 6? ?3 ) }
+
+   condition:
+      #a == 1 and @a[0] == 4 and !a[0] == 3
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
diff --git a/tests/analysis/scan/scanning_str.py b/tests/analysis/scan/scanning_str.py
new file mode 100644
index 0000000..ff36ca3
--- /dev/null
+++ b/tests/analysis/scan/scanning_str.py
@@ -0,0 +1,194 @@
+
+from common import RostTestClass
+from pychrysalide.analysis.contents import MemoryContent
+
+
+class TestRostScanningStrings(RostTestClass):
+    """TestCases for the bytes section syntax (strings)."""
+
+    def testSimpleStringPattern(self):
+        """Test a simple string pattern."""
+
+        cnt = MemoryContent(b'123-Abc-456')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "Abc"
+
+   condition:
+      #a == 1 and @a[0] == 4
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testEscapedStringPattern(self):
+        """Test escaped string patterns."""
+
+        cnt = MemoryContent(b'\a\b\t\n\v\f\r' + bytes([ 0x1b ]) + b'"\\\xff')
+
+        rule = r'''
+rule test {
+
+   strings:
+      $a = "\a\b\t\n\v\f\r\e\"\\\xff"
+
+   condition:
+      #a == 1 and @a[0] == 0
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'\a\b\t\n--123--\v\f\r' + bytes([ 0x1b ]) + b'"\\\xff')
+
+        rule = r'''
+rule test {
+
+   strings:
+      $a = "\a\b\t\n--123--\v\f\r\e\"\\\xff"
+
+   condition:
+      #a == 1 and @a[0] == 0
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testStringModifiers(self):
+        """Check string modifiers output."""
+
+        cnt = MemoryContent(b'--414243--')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "ABC" hex
+
+   condition:
+      #a == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'--ABC--')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "ABC" plain
+
+   condition:
+      #a == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'--CBA--')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "ABC" rev
+
+   condition:
+      #a == 1
+
+}
+'''
+
+
+    def testStringPatternFullword(self):
+        """Test a fullword string pattern."""
+
+        cnt = MemoryContent(b'ABCDEF 123 ')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "DEF" fullword
+      $b = "123" fullword
+
+   condition:
+      #a == 0 and #b == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'DEF 123 ')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "DEF" fullword
+      $b = "123" fullword
+
+   condition:
+      #a == 1 and #b == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+        cnt = MemoryContent(b'\tDEF 123 ')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "DEF" fullword
+      $b = "123" fullword
+
+   condition:
+      #a == 1 and #b == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)
+
+
+    def testStringPatternCase(self):
+        """Test a string pattern with case care."""
+
+        cnt = MemoryContent(b'abc123-Abc123Def456GHI...z0z1z2z3z4z5z6z7z8z9')
+
+        rule = '''
+rule test {
+
+   strings:
+      $a = "Abc" nocase
+      $b = "ABC123DEF456GHI" nocase
+      $z = "z0z1z2z3z4z5z6z7z8z9" nocase
+
+   condition:
+      #a == 2 and #b == 1 and #z == 1
+
+}
+'''
+
+        self.check_rule_success(rule, content=cnt)