Add files via upload

Updated the language in the Driver Signature section for clarity.

Engine/Properties/convert.py

@@ -1,4 +1,4 @@
-import re, os, sys, math, time, zlib, datetime, hashlib
+import re, os, sys, math, time, zlib, mmap, datetime, hashlib
 import ctypes, ctypes.wintypes, pefile, multiprocessing
 import numpy as np
 
@@ -113,6 +113,9 @@ def verify_signature(file_path):
 
 class FeatureExtractor:
     _STRING_PATTERN = re.compile(b'[\x20-\x7E]{5,}')
+    _STRING_FAST_PATH_BYTES = 4194304
+    _STRING_BATCH_BYTES = 4194304
+    _HISTOGRAM_BATCH_WINDOWS = 1024
     
     _BYTE_HIST_KEYS = [f'ByteHist_{i:02X}' for i in range(256)]
     _BYTE_ENT_KEYS = [f'ByteEnt_{i:02X}' for i in range(256)]
@@ -125,6 +128,49 @@ class FeatureExtractor:
     _C_LOG_C_TABLE = np.zeros(2049, dtype=np.float64)
     _C_LOG_C_TABLE[1:] = np.arange(1, 2049) * np.log2(np.arange(1, 2049))
 
+    _OPTIONAL_HEADER_FIELDS = [
+        'Magic', 'MajorLinkerVersion', 'MinorLinkerVersion', 'SizeOfCode',
+        'SizeOfInitializedData', 'SizeOfUninitializedData', 'AddressOfEntryPoint',
+        'BaseOfCode', 'ImageBase', 'SectionAlignment', 'FileAlignment',
+        'MajorOperatingSystemVersion', 'MinorOperatingSystemVersion',
+        'MajorImageVersion', 'MinorImageVersion', 'MajorSubsystemVersion',
+        'MinorSubsystemVersion', 'SizeOfImage', 'SizeOfHeaders', 'CheckSum',
+        'Subsystem', 'DllCharacteristics', 'SizeOfStackReserve', 'SizeOfStackCommit',
+        'SizeOfHeapReserve', 'SizeOfHeapCommit', 'LoaderFlags', 'NumberOfRvaAndSizes'
+    ]
+
+    _SECTION_DEFAULTS = {
+        'SectionCount': 0.0,
+        'SectionMaxEntropy': 0.0,
+        'SectionMinEntropy': 0.0,
+        'SectionMeanEntropy': 0.0,
+        'SectionMaxRawSize': 0.0,
+        'SectionMinRawSize': 0.0,
+        'SectionMeanRawSize': 0.0,
+        'SectionMaxVSize': 0.0,
+        'SectionMinVSize': 0.0,
+        'SectionMeanVSize': 0.0,
+        'ExecutableSections': 0.0,
+        'WritableSections': 0.0,
+        'ReadableSections': 0.0,
+        'SectionException': 0.0
+    }
+
+    _CONDITIONAL_DEFAULTS = {
+        'IsDriver': 0.0,
+        'HasTlsCallbacks': 0.0,
+        'IsDebug': 0.0,
+        'IsPreRelease': 0.0,
+        'IsPatched': 0.0,
+        'IsPrivateBuild': 0.0,
+        'IsSpecialBuild': 0.0,
+        'ImportCount': 0.0,
+        'ImportFunctionCount': 0.0,
+        'ExportCount': 0.0,
+        'IconCount': 0.0,
+        'DebugCount': 0.0
+    }
+
     @staticmethod
     def _safe_float(val):
         try:
@@ -134,6 +180,18 @@ class FeatureExtractor:
         except Exception: 
             return 0.0
 
+    @classmethod
+    def _base_defaults(cls):
+        base = dict.fromkeys(cls._OPTIONAL_HEADER_FIELDS, 0.0)
+        base.update(cls._SECTION_DEFAULTS)
+        base.update(cls._CONDITIONAL_DEFAULTS)
+
+        for i in range(16):
+            base[f'DataDirectory_{i}_Size'] = 0.0
+            base[f'DataDirectory_{i}_VA'] = 0.0
+
+        return base
+
     @staticmethod
     def _calc_entropy(data):
         if not data:
@@ -151,25 +209,84 @@ class FeatureExtractor:
         res = {"StringCount": 0.0, "StringMeanLength": 0.0, "StringEntropy": 0.0}
         res.update(dict.fromkeys(cls._STRING_KEYS, 0.0))
 
-        matches = cls._STRING_PATTERN.findall(file_bytes)
-        if not matches:
-            return res
+        if len(file_bytes) <= cls._STRING_FAST_PATH_BYTES:
+            matches = cls._STRING_PATTERN.findall(file_bytes)
+            if not matches:
+                return res
 
-        combined_data = b''.join(matches)
-        count = len(matches)
-        total_len = len(combined_data)
+            combined_data = b''.join(matches)
+            count = len(matches)
+            total_len = len(combined_data)
+            global_counts = np.bincount(np.frombuffer(combined_data, dtype=np.uint8), minlength=256)
+        else:
+            count = 0
+            total_len = 0
+            global_counts = np.zeros(256, dtype=np.int64)
+            mv = memoryview(file_bytes)
+            pos = 0
+            size = len(file_bytes)
+
+            while pos < size:
+                chunk_end = min(pos + cls._STRING_BATCH_BYTES, size)
+                chunk_arr = np.frombuffer(mv[pos:chunk_end], dtype=np.uint8)
+                separators = (chunk_arr < 0x20) | (chunk_arr > 0x7E)
+
+                if separators.any():
+                    last_separator = len(separators) - 1 - int(np.argmax(separators[::-1]))
+                    segment_end = pos + last_separator + 1
+                    matches = cls._STRING_PATTERN.findall(mv[pos:segment_end])
+
+                    if matches:
+                        combined_data = b''.join(matches)
+                        count += len(matches)
+                        total_len += len(combined_data)
+                        global_counts += np.bincount(np.frombuffer(combined_data, dtype=np.uint8), minlength=256)
+
+                    pos = segment_end
+                    continue
+
+                pending_len = 0
+                pending_counts = np.zeros(256, dtype=np.int64)
+
+                while pos < size:
+                    chunk_end = min(pos + cls._STRING_BATCH_BYTES, size)
+                    chunk_arr = np.frombuffer(mv[pos:chunk_end], dtype=np.uint8)
+                    separators = (chunk_arr < 0x20) | (chunk_arr > 0x7E)
+
+                    if separators.any():
+                        first_separator = int(np.argmax(separators))
+                        if first_separator:
+                            pending_counts += np.bincount(chunk_arr[:first_separator], minlength=256)
+                            pending_len += first_separator
+                        pos += first_separator + 1
+                        break
+
+                    pending_counts += np.bincount(chunk_arr, minlength=256)
+                    pending_len += len(chunk_arr)
+                    pos = chunk_end
+
+                if pending_len >= 5:
+                    count += 1
+                    total_len += pending_len
+                    global_counts += pending_counts
+
+            if count == 0 or total_len == 0:
+                return res
 
         res["StringCount"] = float(count)
         res["StringMeanLength"] = float(total_len) / count
-        res["StringEntropy"] = cls._calc_entropy(combined_data)
 
-        arr = np.frombuffer(combined_data, dtype=np.uint8)
-        counts = np.bincount(arr, minlength=127)
-        char_counts = counts[0x20:0x7F]
+        counts_nonzero = global_counts[global_counts > 0]
+        if len(counts_nonzero) > 0:
+            res["StringEntropy"] = float(np.log2(total_len) - np.sum(counts_nonzero * np.log2(counts_nonzero)) / total_len)
+
+        valid_chars = global_counts[0x20:0x7F]
+        res.update(dict(zip(cls._STRING_KEYS, valid_chars.astype(np.float64) / total_len)))
 
-        res.update(dict(zip(cls._STRING_KEYS, char_counts.astype(np.float64) / total_len)))
         return res
 
+
+
     @classmethod
     def _extract_histograms(cls, file_bytes):
         arr = np.frombuffer(file_bytes, dtype=np.uint8)
@@ -177,56 +294,64 @@ class FeatureExtractor:
         res = {}
 
         if sz == 0:
+            res['FileEntropy'] = 0.0
             res.update(dict.fromkeys(cls._BYTE_HIST_KEYS, 0.0))
             res.update(dict.fromkeys(cls._BYTE_ENT_KEYS, 0.0))
             return res
 
-        byte_counts = np.bincount(arr, minlength=256)
-        byte_hist = byte_counts.astype(np.float64) / sz
-        res.update(dict(zip(cls._BYTE_HIST_KEYS, byte_hist)))
-
+        byte_counts = np.zeros(256, dtype=np.int64)
         ent_hist = np.zeros(256, dtype=np.float64)
         window = 2048
         step = 1024
 
         if sz < window:
-            w_counts = byte_counts
-            p = w_counts[w_counts > 0] / float(sz)
+            byte_counts += np.bincount(arr, minlength=256)
+            p = byte_counts[byte_counts > 0] / float(sz)
             entropy = float(-np.sum(p * np.log2(p)))
             ent_bin = min(int(entropy * 2.0), 15)
-            byte_bins = w_counts.reshape(16, 16).sum(axis=1)
+            byte_bins = byte_counts.reshape(16, 16).sum(axis=1)
             idx_start = ent_bin * 16
             ent_hist[idx_start : idx_start+16] += byte_bins
-
         else:
             num_windows = (sz - window) // step + 1
-            batch_windows = 10000 
-            
-            for w_start in range(0, num_windows, batch_windows):
-                w_end = min(w_start + batch_windows, num_windows)
+
+            for w_start in range(0, num_windows, cls._HISTOGRAM_BATCH_WINDOWS):
+                w_end = min(w_start + cls._HISTOGRAM_BATCH_WINDOWS, num_windows)
                 b_num = w_end - w_start
-                
+
                 start_byte = w_start * step
                 end_byte = start_byte + b_num * step + step
-                b_arr = arr[start_byte:end_byte]
-                
-                arr_2d = b_arr.reshape(b_num + 1, step)
-                offsets = (np.arange(b_num + 1, dtype=np.int32)[:, None] * 256)
-                flat_idx = (arr_2d.astype(np.int32) + offsets).ravel()
-                
-                chunk_counts = np.bincount(flat_idx, minlength=(b_num + 1) * 256).reshape(b_num + 1, 256)
+                encoded = arr[start_byte:end_byte].astype(np.int32).reshape(b_num + 1, step)
+                encoded += np.arange(b_num + 1, dtype=np.int32)[:, None] * 256
+
+                chunk_counts = np.bincount(encoded.ravel(), minlength=(b_num + 1) * 256).reshape(b_num + 1, 256)
+                del encoded
+
+                byte_counts += np.sum(chunk_counts[:-1], axis=0)
+                if w_end == num_windows:
+                    byte_counts += chunk_counts[-1]
+
                 window_counts = chunk_counts[:-1] + chunk_counts[1:]
-                
                 sum_c_log_c = np.sum(cls._C_LOG_C_TABLE[window_counts], axis=1)
                 entropies = 11.0 - sum_c_log_c / 2048.0
-                
+
                 ent_bins = (entropies * 2.0).astype(np.int32)
                 np.clip(ent_bins, 0, 15, out=ent_bins)
-                
+
                 byte_bins = window_counts.reshape(b_num, 16, 16).sum(axis=2)
                 flat_ent_bins = (ent_bins[:, None] * 16 + np.arange(16)).ravel()
                 np.add.at(ent_hist, flat_ent_bins, byte_bins.ravel())
 
+            covered_size = (num_windows + 1) * step
+            if covered_size < sz:
+                byte_counts += np.bincount(arr[covered_size:], minlength=256)
+
+        counts_nonzero = byte_counts[byte_counts > 0]
+        res['FileEntropy'] = float(np.log2(sz) - np.sum(counts_nonzero * np.log2(counts_nonzero)) / sz)
+
+        byte_hist = byte_counts.astype(np.float64) / sz
+        res.update(dict(zip(cls._BYTE_HIST_KEYS, byte_hist)))
+
         ent_sum = np.sum(ent_hist)
         if ent_sum > 0:
             ent_hist /= ent_sum
@@ -234,6 +359,7 @@ class FeatureExtractor:
         res.update(dict(zip(cls._BYTE_ENT_KEYS, ent_hist)))
         return res
 
+
     @classmethod
     def _extract_overlay_features(cls, pe, fsize):
         overlay_offset = pe.get_overlay_data_start_offset()
@@ -390,7 +516,7 @@ class FeatureExtractor:
         if fsize == 0:
             return None
 
-        base, dlls, apis, pe = {}, set(), set(), None
+        base, dlls, apis, pe = cls._base_defaults(), set(), set(), None
         try:
             pe = pefile.PE(data=file_bytes, fast_load=True)
             
@@ -408,7 +534,6 @@ class FeatureExtractor:
                 pass
             
             base['TrustSigned'] = float(verify_signature(file_path))
-            base['FileEntropy'] = cls._calc_entropy(file_bytes)
             base['FileSize'] = float(fsize)
             
             fh = pe.FILE_HEADER
@@ -613,20 +738,22 @@ def _extract_worker_wrapper(args):
     return _extract_worker(*args)
 
 def _extract_worker(file_path, label):
+    mm = None
+    f = None
+
     try:
         fsize = os.path.getsize(file_path)
         if fsize == 0 or fsize > MAX_FILE_SIZE:
             return 'error', None
 
-        with open(file_path, "rb") as f:
-            file_bytes = f.read()
-
-        sha256 = hashlib.sha256(file_bytes).hexdigest()
+        f = open(file_path, "rb")
+        mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+        sha256 = hashlib.sha256(mm).hexdigest()
         
         if sha256 in GLOBAL_HASHES:
             return 'skipped', None
 
-        res = FeatureExtractor.extract(file_bytes, file_path, fsize)
+        res = FeatureExtractor.extract(mm, file_path, fsize)
         if res:
             final_res = {"Label": label, "FileHash": sha256, **res}
             return 'success', (sha256, JSON_DUMPS(final_res) + "\n")
@@ -634,6 +761,11 @@ def _extract_worker(file_path, label):
         return 'error', None
     except Exception:
         return 'error', None
+    finally:
+        if mm:
+            mm.close()
+        if f:
+            f.close()
 
 ####################################################################################################
 

Engine/Properties/scanner.py

@@ -1,4 +1,4 @@
-import os, re, sys, math, json, zlib, pefile, datetime
+import os, re, sys, math, json, zlib, mmap, pefile, datetime
 import ctypes, ctypes.wintypes
 import numpy as np
 import onnxruntime as ort
@@ -99,6 +99,9 @@ class WinTrust:
 
 class FeatureExtractor:
     _STRING_PATTERN = re.compile(b'[\x20-\x7E]{5,}')
+    _STRING_FAST_PATH_BYTES = 4194304
+    _STRING_BATCH_BYTES = 4194304
+    _HISTOGRAM_BATCH_WINDOWS = 1024
     
     _BYTE_HIST_KEYS = [f'ByteHist_{i:02X}' for i in range(256)]
     _BYTE_ENT_KEYS = [f'ByteEnt_{i:02X}' for i in range(256)]
@@ -111,6 +114,49 @@ class FeatureExtractor:
     _C_LOG_C_TABLE = np.zeros(2049, dtype=np.float64)
     _C_LOG_C_TABLE[1:] = np.arange(1, 2049) * np.log2(np.arange(1, 2049))
 
+    _OPTIONAL_HEADER_FIELDS = [
+        'Magic', 'MajorLinkerVersion', 'MinorLinkerVersion', 'SizeOfCode',
+        'SizeOfInitializedData', 'SizeOfUninitializedData', 'AddressOfEntryPoint',
+        'BaseOfCode', 'ImageBase', 'SectionAlignment', 'FileAlignment',
+        'MajorOperatingSystemVersion', 'MinorOperatingSystemVersion',
+        'MajorImageVersion', 'MinorImageVersion', 'MajorSubsystemVersion',
+        'MinorSubsystemVersion', 'SizeOfImage', 'SizeOfHeaders', 'CheckSum',
+        'Subsystem', 'DllCharacteristics', 'SizeOfStackReserve', 'SizeOfStackCommit',
+        'SizeOfHeapReserve', 'SizeOfHeapCommit', 'LoaderFlags', 'NumberOfRvaAndSizes'
+    ]
+
+    _SECTION_DEFAULTS = {
+        'SectionCount': 0.0,
+        'SectionMaxEntropy': 0.0,
+        'SectionMinEntropy': 0.0,
+        'SectionMeanEntropy': 0.0,
+        'SectionMaxRawSize': 0.0,
+        'SectionMinRawSize': 0.0,
+        'SectionMeanRawSize': 0.0,
+        'SectionMaxVSize': 0.0,
+        'SectionMinVSize': 0.0,
+        'SectionMeanVSize': 0.0,
+        'ExecutableSections': 0.0,
+        'WritableSections': 0.0,
+        'ReadableSections': 0.0,
+        'SectionException': 0.0
+    }
+
+    _CONDITIONAL_DEFAULTS = {
+        'IsDriver': 0.0,
+        'HasTlsCallbacks': 0.0,
+        'IsDebug': 0.0,
+        'IsPreRelease': 0.0,
+        'IsPatched': 0.0,
+        'IsPrivateBuild': 0.0,
+        'IsSpecialBuild': 0.0,
+        'ImportCount': 0.0,
+        'ImportFunctionCount': 0.0,
+        'ExportCount': 0.0,
+        'IconCount': 0.0,
+        'DebugCount': 0.0
+    }
+
     @staticmethod
     def _safe_float(val):
         try:
@@ -120,6 +166,18 @@ class FeatureExtractor:
         except Exception: 
             return 0.0
 
+    @classmethod
+    def _base_defaults(cls):
+        base = dict.fromkeys(cls._OPTIONAL_HEADER_FIELDS, 0.0)
+        base.update(cls._SECTION_DEFAULTS)
+        base.update(cls._CONDITIONAL_DEFAULTS)
+
+        for i in range(16):
+            base[f'DataDirectory_{i}_Size'] = 0.0
+            base[f'DataDirectory_{i}_VA'] = 0.0
+
+        return base
+
     @staticmethod
     def _calc_entropy(data):
         if not data:
@@ -137,25 +195,84 @@ class FeatureExtractor:
         res = {"StringCount": 0.0, "StringMeanLength": 0.0, "StringEntropy": 0.0}
         res.update(dict.fromkeys(cls._STRING_KEYS, 0.0))
 
-        matches = cls._STRING_PATTERN.findall(file_bytes)
-        if not matches:
-            return res
+        if len(file_bytes) <= cls._STRING_FAST_PATH_BYTES:
+            matches = cls._STRING_PATTERN.findall(file_bytes)
+            if not matches:
+                return res
 
-        combined_data = b''.join(matches)
-        count = len(matches)
-        total_len = len(combined_data)
+            combined_data = b''.join(matches)
+            count = len(matches)
+            total_len = len(combined_data)
+            global_counts = np.bincount(np.frombuffer(combined_data, dtype=np.uint8), minlength=256)
+        else:
+            count = 0
+            total_len = 0
+            global_counts = np.zeros(256, dtype=np.int64)
+            mv = memoryview(file_bytes)
+            pos = 0
+            size = len(file_bytes)
+
+            while pos < size:
+                chunk_end = min(pos + cls._STRING_BATCH_BYTES, size)
+                chunk_arr = np.frombuffer(mv[pos:chunk_end], dtype=np.uint8)
+                separators = (chunk_arr < 0x20) | (chunk_arr > 0x7E)
+
+                if separators.any():
+                    last_separator = len(separators) - 1 - int(np.argmax(separators[::-1]))
+                    segment_end = pos + last_separator + 1
+                    matches = cls._STRING_PATTERN.findall(mv[pos:segment_end])
+
+                    if matches:
+                        combined_data = b''.join(matches)
+                        count += len(matches)
+                        total_len += len(combined_data)
+                        global_counts += np.bincount(np.frombuffer(combined_data, dtype=np.uint8), minlength=256)
+
+                    pos = segment_end
+                    continue
+
+                pending_len = 0
+                pending_counts = np.zeros(256, dtype=np.int64)
+
+                while pos < size:
+                    chunk_end = min(pos + cls._STRING_BATCH_BYTES, size)
+                    chunk_arr = np.frombuffer(mv[pos:chunk_end], dtype=np.uint8)
+                    separators = (chunk_arr < 0x20) | (chunk_arr > 0x7E)
+
+                    if separators.any():
+                        first_separator = int(np.argmax(separators))
+                        if first_separator:
+                            pending_counts += np.bincount(chunk_arr[:first_separator], minlength=256)
+                            pending_len += first_separator
+                        pos += first_separator + 1
+                        break
+
+                    pending_counts += np.bincount(chunk_arr, minlength=256)
+                    pending_len += len(chunk_arr)
+                    pos = chunk_end
+
+                if pending_len >= 5:
+                    count += 1
+                    total_len += pending_len
+                    global_counts += pending_counts
+
+            if count == 0 or total_len == 0:
+                return res
 
         res["StringCount"] = float(count)
         res["StringMeanLength"] = float(total_len) / count
-        res["StringEntropy"] = cls._calc_entropy(combined_data)
 
-        arr = np.frombuffer(combined_data, dtype=np.uint8)
-        counts = np.bincount(arr, minlength=127)
-        char_counts = counts[0x20:0x7F]
+        counts_nonzero = global_counts[global_counts > 0]
+        if len(counts_nonzero) > 0:
+            res["StringEntropy"] = float(np.log2(total_len) - np.sum(counts_nonzero * np.log2(counts_nonzero)) / total_len)
+
+        valid_chars = global_counts[0x20:0x7F]
+        res.update(dict(zip(cls._STRING_KEYS, valid_chars.astype(np.float64) / total_len)))
 
-        res.update(dict(zip(cls._STRING_KEYS, char_counts.astype(np.float64) / total_len)))
         return res
 
+
+
     @classmethod
     def _extract_histograms(cls, file_bytes):
         arr = np.frombuffer(file_bytes, dtype=np.uint8)
@@ -163,55 +280,64 @@ class FeatureExtractor:
         res = {}
 
         if sz == 0:
+            res['FileEntropy'] = 0.0
             res.update(dict.fromkeys(cls._BYTE_HIST_KEYS, 0.0))
             res.update(dict.fromkeys(cls._BYTE_ENT_KEYS, 0.0))
             return res
 
-        byte_counts = np.bincount(arr, minlength=256)
-        byte_hist = byte_counts.astype(np.float64) / sz
-        res.update(dict(zip(cls._BYTE_HIST_KEYS, byte_hist)))
-
+        byte_counts = np.zeros(256, dtype=np.int64)
         ent_hist = np.zeros(256, dtype=np.float64)
         window = 2048
         step = 1024
 
         if sz < window:
-            w_counts = byte_counts
-            p = w_counts[w_counts > 0] / float(sz)
+            byte_counts += np.bincount(arr, minlength=256)
+            p = byte_counts[byte_counts > 0] / float(sz)
             entropy = float(-np.sum(p * np.log2(p)))
             ent_bin = min(int(entropy * 2.0), 15)
-            byte_bins = w_counts.reshape(16, 16).sum(axis=1)
+            byte_bins = byte_counts.reshape(16, 16).sum(axis=1)
             idx_start = ent_bin * 16
             ent_hist[idx_start : idx_start+16] += byte_bins
         else:
             num_windows = (sz - window) // step + 1
-            batch_windows = 10000 
-            
-            for w_start in range(0, num_windows, batch_windows):
-                w_end = min(w_start + batch_windows, num_windows)
+
+            for w_start in range(0, num_windows, cls._HISTOGRAM_BATCH_WINDOWS):
+                w_end = min(w_start + cls._HISTOGRAM_BATCH_WINDOWS, num_windows)
                 b_num = w_end - w_start
-                
+
                 start_byte = w_start * step
                 end_byte = start_byte + b_num * step + step
-                b_arr = arr[start_byte:end_byte]
-                
-                arr_2d = b_arr.reshape(b_num + 1, step)
-                offsets = (np.arange(b_num + 1, dtype=np.int32)[:, None] * 256)
-                flat_idx = (arr_2d.astype(np.int32) + offsets).ravel()
-                
-                chunk_counts = np.bincount(flat_idx, minlength=(b_num + 1) * 256).reshape(b_num + 1, 256)
+                encoded = arr[start_byte:end_byte].astype(np.int32).reshape(b_num + 1, step)
+                encoded += np.arange(b_num + 1, dtype=np.int32)[:, None] * 256
+
+                chunk_counts = np.bincount(encoded.ravel(), minlength=(b_num + 1) * 256).reshape(b_num + 1, 256)
+                del encoded
+
+                byte_counts += np.sum(chunk_counts[:-1], axis=0)
+                if w_end == num_windows:
+                    byte_counts += chunk_counts[-1]
+
                 window_counts = chunk_counts[:-1] + chunk_counts[1:]
-                
                 sum_c_log_c = np.sum(cls._C_LOG_C_TABLE[window_counts], axis=1)
                 entropies = 11.0 - sum_c_log_c / 2048.0
-                
+
                 ent_bins = (entropies * 2.0).astype(np.int32)
                 np.clip(ent_bins, 0, 15, out=ent_bins)
-                
+
                 byte_bins = window_counts.reshape(b_num, 16, 16).sum(axis=2)
                 flat_ent_bins = (ent_bins[:, None] * 16 + np.arange(16)).ravel()
                 np.add.at(ent_hist, flat_ent_bins, byte_bins.ravel())
 
+            covered_size = (num_windows + 1) * step
+            if covered_size < sz:
+                byte_counts += np.bincount(arr[covered_size:], minlength=256)
+
+        counts_nonzero = byte_counts[byte_counts > 0]
+        res['FileEntropy'] = float(np.log2(sz) - np.sum(counts_nonzero * np.log2(counts_nonzero)) / sz)
+
+        byte_hist = byte_counts.astype(np.float64) / sz
+        res.update(dict(zip(cls._BYTE_HIST_KEYS, byte_hist)))
+
         ent_sum = np.sum(ent_hist)
         if ent_sum > 0:
             ent_hist /= ent_sum
@@ -219,6 +345,7 @@ class FeatureExtractor:
         res.update(dict(zip(cls._BYTE_ENT_KEYS, ent_hist)))
         return res
 
+
     @classmethod
     def _extract_overlay_features(cls, pe, fsize):
         overlay_offset = pe.get_overlay_data_start_offset()
@@ -372,15 +499,19 @@ class FeatureExtractor:
     @classmethod
     def extract(cls, file_path):
         pe = None
+        mm = None
+        source_file = None
+
         try:
             fsize = os.path.getsize(file_path)
             if fsize == 0 or fsize > MAX_FILE_SIZE: 
                 return None
 
-            with open(file_path, "rb") as f:
-                file_bytes = f.read()
+            source_file = open(file_path, "rb")
+            mm = mmap.mmap(source_file.fileno(), 0, access=mmap.ACCESS_READ)
+            file_bytes = mm
 
-            base, dlls, apis = {}, set(), set()
+            base, dlls, apis = cls._base_defaults(), set(), set()
             pe = pefile.PE(data=file_bytes, fast_load=True)
 
             try:
@@ -397,7 +528,6 @@ class FeatureExtractor:
                 pass
             
             base['TrustSigned'] = float(WinTrust.verify(file_path))
-            base['FileEntropy'] = cls._calc_entropy(file_bytes)
             base['FileSize'] = float(fsize)
             
             fh = pe.FILE_HEADER
@@ -420,7 +550,8 @@ class FeatureExtractor:
             if hasattr(pe, 'OPTIONAL_HEADER'):
                 op = pe.OPTIONAL_HEADER
                 fields = ['Magic', 'MajorLinkerVersion', 'MinorLinkerVersion', 'SizeOfCode', 'SizeOfInitializedData', 'SizeOfUninitializedData', 'AddressOfEntryPoint', 'BaseOfCode', 'ImageBase', 'SectionAlignment', 'FileAlignment', 'MajorOperatingSystemVersion', 'MinorOperatingSystemVersion', 'MajorImageVersion', 'MinorImageVersion', 'MajorSubsystemVersion', 'MinorSubsystemVersion', 'SizeOfImage', 'SizeOfHeaders', 'CheckSum', 'Subsystem', 'DllCharacteristics', 'SizeOfStackReserve', 'SizeOfStackCommit', 'SizeOfHeapReserve', 'SizeOfHeapCommit', 'LoaderFlags', 'NumberOfRvaAndSizes']
-                for f in fields: base[f] = getattr(op, f, 0)
+                for field_name in fields:
+                    base[field_name] = getattr(op, field_name, 0)
                 base['IsDriver'] = 1.0 if base.get('Subsystem') == 1 else 0.0
 
                 if hasattr(op, 'DATA_DIRECTORY'):
@@ -571,11 +702,24 @@ class FeatureExtractor:
 
             return {"Base": base, "DLLs": list(dlls), "APIs": list(apis)}
 
-        except Exception:
+        except pefile.PEFormatError:
             return None
         finally:
-            if pe:
-                pe.close()
+            if pe is not None:
+                try:
+                    pe.close()
+                except Exception:
+                    pass
+            if mm is not None:
+                try:
+                    mm.close()
+                except Exception:
+                    pass
+            if source_file is not None:
+                try:
+                    source_file.close()
+                except Exception:
+                    pass
 
 ####################################################################################################
 

Engine/Properties/train_server.py

@@ -58,15 +58,14 @@ def analyze_schema(jsonl_path):
         for line in f:
             num_lines += 1
 
-            if not base_keys:
-                try:
-                    data = JSON_LOADS(line)
-                    base_data = data.get('Base')
-                    if base_data:
-                        base_keys.update(base_data.keys())
+            try:
+                data = JSON_LOADS(line)
+                base_data = data.get('Base')
+                if base_data:
+                    base_keys.update(base_data.keys())
 
-                except Exception:
-                    pass
+            except Exception:
+                pass
 
     base_schema = sorted([k for k in base_keys if k.lower().strip() not in EXCLUDE_FEATURES])
     dll_schema = [f"DllHash_{i:03d}" for i in range(DLL_HASH_DIM)]

Engine/Properties/train_user.py

@@ -58,15 +58,14 @@ def analyze_schema(jsonl_path):
         for line in f:
             num_lines += 1
 
-            if not base_keys:
-                try:
-                    data = JSON_LOADS(line)
-                    base_data = data.get('Base')
-                    if base_data:
-                        base_keys.update(base_data.keys())
+            try:
+                data = JSON_LOADS(line)
+                base_data = data.get('Base')
+                if base_data:
+                    base_keys.update(base_data.keys())
 
-                except Exception:
-                    pass
+            except Exception:
+                pass
 
     base_schema = sorted([k for k in base_keys if k.lower().strip() not in EXCLUDE_FEATURES])
     dll_schema = [f"DllHash_{i:03d}" for i in range(DLL_HASH_DIM)]

Interface/templates/index.html

@@ -319,7 +319,7 @@
                         <div class="license-widget">
                             <textarea class="license-text" readonly>87owo Publicly Available Source Code License Agreement
 
-Last Updated: 2026-06-09
+Last Updated: 2026-07-18
 IMPORTANT NOTICE: THIS LICENSE IS "SOURCE AVAILABLE" AND IS NOT AN OPEN SOURCE LICENSE APPROVED BY THE OPEN SOURCE INITIATIVE (OSI). BY DOWNLOADING, ACCESSING, OR USING THE SOFTWARE, YOU AGREE TO BE BOUND BY THESE TERMS.
 
 1. Definitions and Interpretation
@@ -371,6 +371,7 @@ IMPORTANT NOTICE: THIS LICENSE IS "SOURCE AVAILABLE" AND IS NOT AN OPEN SOURCE L
    - 7.3. Scope of Collected Extensions: Sample collection targets potential executable and system binary formats, which explicitly include, but are not limited to, files with the following extensions: `.exe`, `.dll`, `.sys`, `.ocx`, `.scr`, `.efi`, `.acm`, `.ax`, `.cpl`, `.drv`, `.com`, `.mui`, `.pyd`, `.wfx`, `.api`, `.awx`, `.rll`, and `.winmd`.
    - 7.4. Public Disclosure of Threat Intelligence: You explicitly acknowledge and agree that the metadata, behavioral reports, and threat intelligence data generated from the cloud analysis of submitted samples may be publicly disclosed, published, or shared with the cybersecurity community for collaborative defense purposes.
    - 7.5. Opt-Out Mechanism: Cloud-based telemetry and sample submission are fully optional. Users who do not wish to participate in sample collection or cloud-assisted analysis may disable the corresponding cloud services at any time via the toggle switches provided within the Software’s settings interface.
+   - 7.6. Driver Signature: Certain kernel-mode drivers distributed with this software may be digitally signed using code signing methods that are not certified by Microsoft Windows Hardware Quality Labs (WHQL). By installing or using this software, you acknowledge this and decide for yourself whether to install and trust such drivers on your system.
 
 8. Warranty Disclaimer and Limitation of Liability
 

LICENSE.txt

@@ -1,6 +1,6 @@
 87owo Publicly Available Source Code License Agreement
 
-Last Updated: 2026-06-09
+Last Updated: 2026-07-18
 IMPORTANT NOTICE: THIS LICENSE IS "SOURCE AVAILABLE" AND IS NOT AN OPEN SOURCE LICENSE APPROVED BY THE OPEN SOURCE INITIATIVE (OSI). BY DOWNLOADING, ACCESSING, OR USING THE SOFTWARE, YOU AGREE TO BE BOUND BY THESE TERMS.
 
 1. Definitions and Interpretation
@@ -52,6 +52,7 @@ IMPORTANT NOTICE: THIS LICENSE IS "SOURCE AVAILABLE" AND IS NOT AN OPEN SOURCE L
    - 7.3. Scope of Collected Extensions: Sample collection targets potential executable and system binary formats, which explicitly include, but are not limited to, files with the following extensions: `.exe`, `.dll`, `.sys`, `.ocx`, `.scr`, `.efi`, `.acm`, `.ax`, `.cpl`, `.drv`, `.com`, `.mui`, `.pyd`, `.wfx`, `.api`, `.awx`, `.rll`, and `.winmd`.
    - 7.4. Public Disclosure of Threat Intelligence: You explicitly acknowledge and agree that the metadata, behavioral reports, and threat intelligence data generated from the cloud analysis of submitted samples may be publicly disclosed, published, or shared with the cybersecurity community for collaborative defense purposes.
    - 7.5. Opt-Out Mechanism: Cloud-based telemetry and sample submission are fully optional. Users who do not wish to participate in sample collection or cloud-assisted analysis may disable the corresponding cloud services at any time via the toggle switches provided within the Software’s settings interface.
+   - 7.6. Driver Signature: Certain kernel-mode drivers distributed with this software may be digitally signed using code signing methods that are not certified by Microsoft Windows Hardware Quality Labs (WHQL). By installing or using this software, you acknowledge this and decide for yourself whether to install and trust such drivers on your system.
 
 8. Warranty Disclaimer and Limitation of Liability
 

PYAS.py

@@ -296,6 +296,8 @@ class _MainMixin:
         self.scan_count = 0
         self.scan_events = {}
         self.hash_cache = {}
+        self.file_task_timers = {}
+        self.file_task_generations = {}
         self.mbr_backup = {}
         self.cloud_pending = set()
         self.last_io_counters = {}
@@ -313,7 +315,7 @@ class _MainMixin:
         self.lock_io = threading.RLock()
 
         self.pyas_default = {
-            "version": "3.6.3",
+            "version": "3.6.4",
             "api_host": "https://pyas-security.com/",
             "api_key": "fBRZxYS1UxykM-qzNOlKOEl63WILzlvgNMn6QfsG6FXCAAIktCrOPTAfY5_hEyuZ",
             "suffix": [".exe", ".dll", ".sys", ".ocx", ".scr", ".efi", ".acm", ".ax", ".cpl", ".drv", ".com", ".mui", ".pyd", ".wfx", ".api", ".awx", ".rll", ".winmd"],
@@ -475,6 +477,7 @@ class _MainMixin:
                     self.manage_autostart(False)
 
                 elif key == "document_switch":
+                    self._cancel_pending_file_scans()
                     with self.lock_file_ops:
                         if getattr(self, 'h_dir_file', None):
                             try:
@@ -839,6 +842,8 @@ class _MainMixin:
             self.pyas_config["driver_switch"] = False
             self.pyas_config["network_switch"] = False
 
+        self._cancel_pending_file_tasks()
+
         with self.lock_file_ops:
             if getattr(self, 'h_dir_file', None):
                 try:
@@ -1249,21 +1254,14 @@ def start_api(port_container, ready_event):
 if __name__ == "__main__":
     if "-driver-unload" in sys.argv or "-driver-uninstall" in sys.argv:
         controller = WindowAPI()
-        success = controller.stop_system_driver()
         if "-driver-uninstall" in sys.argv:
-            deleted, delete_error = False, 0
-            for _ in range(20):
-                deleted, delete_error = controller._delete_driver_service()
-                if deleted:
-                    break
-                if delete_error not in (1051, 1072):
-                    break
-                time.sleep(0.25)
-            if not deleted:
+            success, delete_error = controller.uninstall_system_driver()
+            if not success:
                 controller.write_log("WARN", "Driver Service", detail=f"DeleteService failed: 0x{delete_error & 0xFFFFFFFF:08X}", success=False)
-            success = (success or not controller.check_system_driver()) and deleted
+        else:
+            success = controller.stop_system_driver()
         controller.flush_logs_now()
-        os._exit(0 if success or not controller.check_system_driver() else 2)
+        os._exit(0 if success else 2)
 
     hide_on_start = "-h" in sys.argv or "-hide" in sys.argv
     init_width, init_height = 980, 670

PYAS_Engine.py

@@ -189,6 +189,9 @@ class rule_scanner:
 
 class pe_scanner:
     _STRING_PATTERN = re.compile(b'[\x20-\x7E]{5,}')
+    _STRING_FAST_PATH_BYTES = 4194304
+    _STRING_BATCH_BYTES = 4194304
+    _HISTOGRAM_BATCH_WINDOWS = 1024
 
     _BYTE_HIST_KEYS = [f'ByteHist_{i:02X}' for i in range(256)]
     _BYTE_ENT_KEYS = [f'ByteEnt_{i:02X}' for i in range(256)]
@@ -202,6 +205,49 @@ class pe_scanner:
     _C_LOG_C_TABLE = numpy.zeros(2049, dtype=numpy.float64)
     _C_LOG_C_TABLE[1:] = numpy.arange(1, 2049) * numpy.log2(numpy.arange(1, 2049))
 
+    _OPTIONAL_HEADER_FIELDS = [
+        'Magic', 'MajorLinkerVersion', 'MinorLinkerVersion', 'SizeOfCode',
+        'SizeOfInitializedData', 'SizeOfUninitializedData', 'AddressOfEntryPoint',
+        'BaseOfCode', 'ImageBase', 'SectionAlignment', 'FileAlignment',
+        'MajorOperatingSystemVersion', 'MinorOperatingSystemVersion',
+        'MajorImageVersion', 'MinorImageVersion', 'MajorSubsystemVersion',
+        'MinorSubsystemVersion', 'SizeOfImage', 'SizeOfHeaders', 'CheckSum',
+        'Subsystem', 'DllCharacteristics', 'SizeOfStackReserve', 'SizeOfStackCommit',
+        'SizeOfHeapReserve', 'SizeOfHeapCommit', 'LoaderFlags', 'NumberOfRvaAndSizes'
+    ]
+
+    _SECTION_DEFAULTS = {
+        'SectionCount': 0.0,
+        'SectionMaxEntropy': 0.0,
+        'SectionMinEntropy': 0.0,
+        'SectionMeanEntropy': 0.0,
+        'SectionMaxRawSize': 0.0,
+        'SectionMinRawSize': 0.0,
+        'SectionMeanRawSize': 0.0,
+        'SectionMaxVSize': 0.0,
+        'SectionMinVSize': 0.0,
+        'SectionMeanVSize': 0.0,
+        'ExecutableSections': 0.0,
+        'WritableSections': 0.0,
+        'ReadableSections': 0.0,
+        'SectionException': 0.0
+    }
+
+    _CONDITIONAL_DEFAULTS = {
+        'IsDriver': 0.0,
+        'HasTlsCallbacks': 0.0,
+        'IsDebug': 0.0,
+        'IsPreRelease': 0.0,
+        'IsPatched': 0.0,
+        'IsPrivateBuild': 0.0,
+        'IsSpecialBuild': 0.0,
+        'ImportCount': 0.0,
+        'ImportFunctionCount': 0.0,
+        'ExportCount': 0.0,
+        'IconCount': 0.0,
+        'DebugCount': 0.0
+    }
+
     def __init__(self):
         self.model = None
         self.input_name = None
@@ -276,6 +322,17 @@ class pe_scanner:
         except Exception:
             return 0.0
 
+    def _base_defaults(self):
+        base = dict.fromkeys(self._OPTIONAL_HEADER_FIELDS, 0.0)
+        base.update(self._SECTION_DEFAULTS)
+        base.update(self._CONDITIONAL_DEFAULTS)
+
+        for i in range(16):
+            base[f'DataDirectory_{i}_Size'] = 0.0
+            base[f'DataDirectory_{i}_VA'] = 0.0
+
+        return base
+
     def _calc_entropy(self, data):
         if not data: 
             return 0.0
@@ -291,20 +348,69 @@ class pe_scanner:
         res = {"StringCount": 0.0, "StringMeanLength": 0.0, "StringEntropy": 0.0}
         res.update(dict.fromkeys(self._STRING_KEYS, 0.0))
 
-        total_len = 0
-        count = 0
-        global_counts = numpy.zeros(256, dtype=numpy.int64)
-        mv = memoryview(file_bytes)
+        if len(file_bytes) <= self._STRING_FAST_PATH_BYTES:
+            matches = self._STRING_PATTERN.findall(file_bytes)
+            if not matches:
+                return res
 
-        for m in self._STRING_PATTERN.finditer(file_bytes):
-            count += 1
-            start, end = m.span()
-            total_len += (end - start)
-            arr = numpy.frombuffer(mv[start:end], dtype=numpy.uint8)
-            global_counts += numpy.bincount(arr, minlength=256)
+            combined_data = b''.join(matches)
+            count = len(matches)
+            total_len = len(combined_data)
+            global_counts = numpy.bincount(numpy.frombuffer(combined_data, dtype=numpy.uint8), minlength=256)
+        else:
+            count = 0
+            total_len = 0
+            global_counts = numpy.zeros(256, dtype=numpy.int64)
+            mv = memoryview(file_bytes)
+            pos = 0
+            size = len(file_bytes)
 
-        if count == 0 or total_len == 0:
-            return res
+            while pos < size:
+                chunk_end = min(pos + self._STRING_BATCH_BYTES, size)
+                chunk_arr = numpy.frombuffer(mv[pos:chunk_end], dtype=numpy.uint8)
+                separators = (chunk_arr < 0x20) | (chunk_arr > 0x7E)
+
+                if separators.any():
+                    last_separator = len(separators) - 1 - int(numpy.argmax(separators[::-1]))
+                    segment_end = pos + last_separator + 1
+                    matches = self._STRING_PATTERN.findall(mv[pos:segment_end])
+
+                    if matches:
+                        combined_data = b''.join(matches)
+                        count += len(matches)
+                        total_len += len(combined_data)
+                        global_counts += numpy.bincount(numpy.frombuffer(combined_data, dtype=numpy.uint8), minlength=256)
+
+                    pos = segment_end
+                    continue
+
+                pending_len = 0
+                pending_counts = numpy.zeros(256, dtype=numpy.int64)
+
+                while pos < size:
+                    chunk_end = min(pos + self._STRING_BATCH_BYTES, size)
+                    chunk_arr = numpy.frombuffer(mv[pos:chunk_end], dtype=numpy.uint8)
+                    separators = (chunk_arr < 0x20) | (chunk_arr > 0x7E)
+
+                    if separators.any():
+                        first_separator = int(numpy.argmax(separators))
+                        if first_separator:
+                            pending_counts += numpy.bincount(chunk_arr[:first_separator], minlength=256)
+                            pending_len += first_separator
+                        pos += first_separator + 1
+                        break
+
+                    pending_counts += numpy.bincount(chunk_arr, minlength=256)
+                    pending_len += len(chunk_arr)
+                    pos = chunk_end
+
+                if pending_len >= 5:
+                    count += 1
+                    total_len += pending_len
+                    global_counts += pending_counts
+
+            if count == 0 or total_len == 0:
+                return res
 
         res["StringCount"] = float(count)
         res["StringMeanLength"] = float(total_len) / count
@@ -318,62 +424,72 @@ class pe_scanner:
 
         return res
 
+
+
     def _extract_histograms(self, file_bytes):
         arr = numpy.frombuffer(file_bytes, dtype=numpy.uint8)
         sz = len(arr)
         res = {}
 
         if sz == 0:
+            res['FileEntropy'] = 0.0
             res.update(dict.fromkeys(self._BYTE_HIST_KEYS, 0.0))
             res.update(dict.fromkeys(self._BYTE_ENT_KEYS, 0.0))
             return res
 
-        byte_counts = numpy.bincount(arr, minlength=256)
-        byte_hist = byte_counts.astype(numpy.float64) / sz
-        res.update(dict(zip(self._BYTE_HIST_KEYS, byte_hist)))
-
+        byte_counts = numpy.zeros(256, dtype=numpy.int64)
         ent_hist = numpy.zeros(256, dtype=numpy.float64)
         window = 2048
         step = 1024
 
         if sz < window:
-            w_counts = byte_counts
-            p = w_counts[w_counts > 0] / float(sz)
+            byte_counts += numpy.bincount(arr, minlength=256)
+            p = byte_counts[byte_counts > 0] / float(sz)
             entropy = float(-numpy.sum(p * numpy.log2(p)))
             ent_bin = min(int(entropy * 2.0), 15)
-            byte_bins = w_counts.reshape(16, 16).sum(axis=1)
+            byte_bins = byte_counts.reshape(16, 16).sum(axis=1)
             idx_start = ent_bin * 16
             ent_hist[idx_start : idx_start+16] += byte_bins
-
         else:
             num_windows = (sz - window) // step + 1
-            batch_windows = 10000 
-            
-            for w_start in range(0, num_windows, batch_windows):
-                w_end = min(w_start + batch_windows, num_windows)
+
+            for w_start in range(0, num_windows, self._HISTOGRAM_BATCH_WINDOWS):
+                w_end = min(w_start + self._HISTOGRAM_BATCH_WINDOWS, num_windows)
                 b_num = w_end - w_start
-                
+
                 start_byte = w_start * step
                 end_byte = start_byte + b_num * step + step
-                b_arr = arr[start_byte:end_byte]
-                
-                arr_2d = b_arr.reshape(b_num + 1, step)
-                offsets = (numpy.arange(b_num + 1, dtype=numpy.int32)[:, None] * 256)
-                flat_idx = (arr_2d.astype(numpy.int32) + offsets).ravel()
-                
-                chunk_counts = numpy.bincount(flat_idx, minlength=(b_num + 1) * 256).reshape(b_num + 1, 256)
+                encoded = arr[start_byte:end_byte].astype(numpy.int32).reshape(b_num + 1, step)
+                encoded += numpy.arange(b_num + 1, dtype=numpy.int32)[:, None] * 256
+
+                chunk_counts = numpy.bincount(encoded.ravel(), minlength=(b_num + 1) * 256).reshape(b_num + 1, 256)
+                del encoded
+
+                byte_counts += numpy.sum(chunk_counts[:-1], axis=0)
+                if w_end == num_windows:
+                    byte_counts += chunk_counts[-1]
+
                 window_counts = chunk_counts[:-1] + chunk_counts[1:]
-                
                 sum_c_log_c = numpy.sum(self._C_LOG_C_TABLE[window_counts], axis=1)
                 entropies = 11.0 - sum_c_log_c / 2048.0
-                
+
                 ent_bins = (entropies * 2.0).astype(numpy.int32)
                 numpy.clip(ent_bins, 0, 15, out=ent_bins)
-                
+
                 byte_bins = window_counts.reshape(b_num, 16, 16).sum(axis=2)
                 flat_ent_bins = (ent_bins[:, None] * 16 + numpy.arange(16)).ravel()
                 numpy.add.at(ent_hist, flat_ent_bins, byte_bins.ravel())
 
+            covered_size = (num_windows + 1) * step
+            if covered_size < sz:
+                byte_counts += numpy.bincount(arr[covered_size:], minlength=256)
+
+        counts_nonzero = byte_counts[byte_counts > 0]
+        res['FileEntropy'] = float(numpy.log2(sz) - numpy.sum(counts_nonzero * numpy.log2(counts_nonzero)) / sz)
+
+        byte_hist = byte_counts.astype(numpy.float64) / sz
+        res.update(dict(zip(self._BYTE_HIST_KEYS, byte_hist)))
+
         ent_sum = numpy.sum(ent_hist)
         if ent_sum > 0:
             ent_hist /= ent_sum
@@ -381,6 +497,7 @@ class pe_scanner:
         res.update(dict(zip(self._BYTE_ENT_KEYS, ent_hist)))
         return res
 
+
     def _extract_overlay_features(self, pe, fsize):
         overlay_offset = pe.get_overlay_data_start_offset()
         if not overlay_offset or overlay_offset >= fsize:
@@ -539,7 +656,7 @@ class pe_scanner:
             f = open(file_path, "rb")
             mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
 
-            base = {}
+            base = self._base_defaults()
             dlls = set()
             apis = set()
             pe = pefile.PE(data=mm, fast_load=True)
@@ -558,7 +675,6 @@ class pe_scanner:
                 pass
             
             base['TrustSigned'] = 1.0 if self.signer.sign_verify(file_path) else 0.0
-            base['FileEntropy'] = self._calc_entropy(mm)
             base['FileSize'] = float(fsize)
             
             fh = pe.FILE_HEADER

PYAS_Inno.iss

@@ -1,6 +1,6 @@
 #define AppId "{{a7d7bac3-93b8-4630-8308-c7a56bf7fdf4}"
 #define AppName "PYAS"
-#define AppVersion "3.6.3.0"
+#define AppVersion "3.6.4.0"
 #define AppPublisher "PYAS Security"
 #define AppURL "https://github.com/87owo/PYAS"
 #define AppExeName "PYAS.exe"
@@ -59,7 +59,11 @@ Name: "install_webview2"; Description: "{cm:InstallWebView2}"; GroupDescription:
 Name: "install_vcredist"; Description: "{cm:InstallVCRedist}"; GroupDescription: "{cm:Dependencies}"
 
 [Files]
-Source: "Payload\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
+Source: "Payload\Engine\*"; DestDir: "{app}\Engine"; Flags: ignoreversion recursesubdirs createallsubdirs
+Source: "Payload\Interface\*"; DestDir: "{app}\Interface"; Flags: ignoreversion recursesubdirs createallsubdirs
+Source: "Payload\License\*"; DestDir: "{app}\License"; Flags: ignoreversion recursesubdirs createallsubdirs
+Source: "Payload\PYAS.exe"; DestDir: "{app}"; Flags: ignoreversion
+Source: "Payload\Plugins\*"; DestDir: "{app}\Plugins"; Flags: ignoreversion recursesubdirs createallsubdirs
 Source: "Redist\VC_redist.x64.exe"; DestDir: "{tmp}\PYAS_Redist"; Flags: deleteafterinstall ignoreversion
 Source: "Redist\MicrosoftEdgeWebview2Setup.exe"; DestDir: "{tmp}\PYAS_Redist"; Flags: deleteafterinstall ignoreversion
 

PYAS_Protect.py

@@ -7,8 +7,18 @@ FLT_PORT_FLAG_SYNC_HANDLE = 0x00000001
 HRESULT_IO_PENDING = 0x800703E5
 ERROR_OPERATION_ABORTED = 995
 ERROR_NOT_FOUND = 1168
+ERROR_SHARING_VIOLATION = 32
+ERROR_LOCK_VIOLATION = 33
 WAIT_OBJECT_0 = 0
 WAIT_TIMEOUT = 258
+INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
+GENERIC_READ = 0x80000000
+FILE_SHARE_READ = 0x00000001
+OPEN_EXISTING = 3
+FILE_ATTRIBUTE_NORMAL = 0x00000080
+FILE_SCAN_DEBOUNCE_SECONDS = 0.35
+FILE_SCAN_RETRY_SECONDS = 0.25
+FILE_LOCK_RETRY_SECONDS = 0.5
 PYAS_CONNECTION_MAGIC = 0x53415950
 PYAS_CONNECTION_VERSION = 1
 PROCESS_TERMINATE = 0x0001
@@ -94,75 +104,98 @@ class ProtectMixin:
 
 ####################################################################################################
 
-    def lock_file(self, target_path, lock, quiet=False):
+    def _try_lock_file(self, file_path):
+        fd = None
+        try:
+            fd = os.open(file_path, os.O_RDWR | os.O_BINARY)
+            try:
+                size = os.path.getsize(file_path)
+            except Exception:
+                size = 1
+
+            lock_size = size if size > 0 else 1
+            msvcrt.locking(fd, msvcrt.LK_NBRLCK, lock_size)
+            self.virus_lock[file_path] = (fd, lock_size)
+            return True, None
+        except Exception as e:
+            if fd is not None:
+                try:
+                    os.close(fd)
+                except Exception:
+                    pass
+            return False, e
+
+    def _queue_file_lock(self, file_path):
+        self._schedule_file_task("lock", file_path, self._retry_file_lock, FILE_LOCK_RETRY_SECONDS)
+
+    def _retry_file_lock(self, file_path):
+        if not os.path.isfile(file_path):
+            return
+
         with self.lock_file_ops:
-            if lock:
-                if not os.path.exists(target_path):
-                    return
+            if file_path in self.virus_lock:
+                return
+            success, _ = self._try_lock_file(file_path)
 
-                paths_to_lock = []
-                if os.path.isdir(target_path):
-                    for root, _, files in os.walk(target_path):
-                        for f in files:
-                            paths_to_lock.append(os.path.join(root, f))
-                else:
-                    paths_to_lock.append(target_path)
+        if not success:
+            self._queue_file_lock(file_path)
 
+    def lock_file(self, target_path, lock, quiet=False):
+        if lock:
+            if not os.path.exists(target_path):
+                return False
+
+            paths_to_lock = []
+            if os.path.isdir(target_path):
+                for root, _, files in os.walk(target_path):
+                    for file_name in files:
+                        paths_to_lock.append(os.path.join(root, file_name))
+            else:
+                paths_to_lock.append(target_path)
+
+            failed_locks = []
+            with self.lock_file_ops:
                 for file_path in paths_to_lock:
                     if file_path in self.virus_lock:
                         continue
 
-                    success = False
-                    last_error = None
-                    for _ in range(5):
-                        try:
-                            fd = os.open(file_path, os.O_RDWR | os.O_BINARY)
-                            try:
-                                try:
-                                    size = os.path.getsize(file_path)
-                                except Exception:
-                                    size = 1
+                    success, error = self._try_lock_file(file_path)
+                    if not success:
+                        failed_locks.append((file_path, error))
 
-                                lock_size = size if size > 0 else 1
-                                msvcrt.locking(fd, msvcrt.LK_NBRLCK, lock_size)
-                                self.virus_lock[file_path] = (fd, lock_size)
-                                success = True
-                                break
+            for file_path, error in failed_locks:
+                self._queue_file_lock(file_path)
+                if not quiet:
+                    self.write_log("INFO", "File Lock Deferred", source=file_path, detail=str(error))
 
-                            except Exception as e:
-                                os.close(fd)
-                                raise e
+            return not failed_locks
 
-                        except Exception as e:
-                            last_error = e
-                            time.sleep(0.1)
+        self._cancel_pending_file_tasks("lock", target_path)
+        with self.lock_file_ops:
+            target_norm = os.path.normcase(target_path)
+            target_dir = target_norm + os.sep
+            keys_to_unlock = []
 
-                    if not success and not quiet:
-                        self.write_log("WARN", "lock_file", source=file_path, detail=str(last_error), success=False)
-            else:
-                target_norm = os.path.normcase(target_path)
-                target_dir = target_norm + os.sep
-                keys_to_unlock = []
+            for locked_path in self.virus_lock:
+                locked_norm = os.path.normcase(locked_path)
+                if locked_norm == target_norm or locked_norm.startswith(target_dir):
+                    keys_to_unlock.append(locked_path)
 
-                for locked_path in self.virus_lock:
-                    locked_norm = os.path.normcase(locked_path)
-                    if locked_norm == target_norm or locked_norm.startswith(target_dir):
-                        keys_to_unlock.append(locked_path)
-
-                for file_path in keys_to_unlock:
-                    fd, lock_size = self.virus_lock[file_path]
+            for file_path in keys_to_unlock:
+                fd, lock_size = self.virus_lock[file_path]
+                try:
+                    msvcrt.locking(fd, msvcrt.LK_UNLCK, lock_size)
+                except Exception as e:
+                    if not quiet:
+                        self.write_log("WARN", "unlock_file", source=file_path, detail=str(e), success=False)
+                finally:
                     try:
-                        msvcrt.locking(fd, msvcrt.LK_UNLCK, lock_size)
-                    except Exception as e:
-                        if not quiet:
-                            self.write_log("WARN", "unlock_file", source=file_path, detail=str(e), success=False)
+                        os.close(fd)
+                    except Exception:
+                        pass
+                    del self.virus_lock[file_path]
 
-                    finally:
-                        try:
-                            os.close(fd)
-                        except Exception:
-                            pass
-                        del self.virus_lock[file_path]
+        return True
 
     def relock_file(self):
         while True:
@@ -665,7 +698,7 @@ class ProtectMixin:
                                         suffix = self.pyas_config.get("suffix", [])
 
                                     if not ext_filter or os.path.splitext(file_path)[-1].lower() in suffix:
-                                        self.protect_pool.submit(self.handle_new_file, file_path)
+                                        self._queue_file_scan(file_path)
 
                         if notify.NextEntryOffset == 0:
                             break
@@ -683,33 +716,106 @@ class ProtectMixin:
                         pass
                     self.h_dir_file = None
 
+    def _cancel_pending_file_tasks(self, task_name=None, target_path=None):
+        target_norm = os.path.normcase(target_path) if target_path else None
+        target_dir = target_norm + os.sep if target_norm else None
+
+        with self.lock_file_ops:
+            task_keys = []
+            for task_key in self.file_task_timers:
+                name, path_key = task_key
+                if task_name and name != task_name:
+                    continue
+                if target_norm and path_key != target_norm and not path_key.startswith(target_dir):
+                    continue
+                task_keys.append(task_key)
+
+            timers = [self.file_task_timers.pop(task_key) for task_key in task_keys]
+            for task_key in task_keys:
+                self.file_task_generations.pop(task_key, None)
+
+        for timer in timers:
+            timer.cancel()
+
+    def _cancel_pending_file_scans(self):
+        self._cancel_pending_file_tasks("scan")
+
+    def _schedule_file_task(self, task_name, file_path, callback, delay):
+        task_key = (task_name, os.path.normcase(file_path))
+
+        with self.lock_file_ops:
+            previous_timer = self.file_task_timers.get(task_key)
+            if previous_timer:
+                previous_timer.cancel()
+
+            generation = self.file_task_generations.get(task_key, 0) + 1
+            timer = threading.Timer(delay, self._run_file_task, args=(task_key, file_path, generation, callback))
+            timer.daemon = True
+            self.file_task_timers[task_key] = timer
+            self.file_task_generations[task_key] = generation
+
+        timer.start()
+
+    def _run_file_task(self, task_key, file_path, generation, callback):
+        with self.lock_file_ops:
+            if self.file_task_generations.get(task_key) != generation:
+                return
+            if self.file_task_timers.get(task_key) is threading.current_thread():
+                self.file_task_timers.pop(task_key, None)
+
+        try:
+            callback(file_path)
+        finally:
+            with self.lock_file_ops:
+                if self.file_task_generations.get(task_key) == generation:
+                    self.file_task_timers.pop(task_key, None)
+                    self.file_task_generations.pop(task_key, None)
+
+    def _queue_file_scan(self, file_path, delay=FILE_SCAN_DEBOUNCE_SECONDS):
+        self._schedule_file_task("scan", file_path, self.handle_new_file, delay)
+
+    def _acquire_file_scan_gate(self, file_path):
+        ctypes.set_last_error(0)
+        handle = self.kernel32.CreateFileW(
+            file_path,
+            GENERIC_READ,
+            FILE_SHARE_READ,
+            None,
+            OPEN_EXISTING,
+            FILE_ATTRIBUTE_NORMAL,
+            None
+        )
+
+        if handle == INVALID_HANDLE_VALUE:
+            return None, ctypes.get_last_error()
+
+        return handle, 0
+
     def handle_new_file(self, file_path):
         try:
-            for _ in range(5):
-                try:
-                    fd = os.open(file_path, os.O_RDONLY | os.O_BINARY)
-                    os.close(fd)
+            with self.lock_config:
+                if not self.pyas_config.get("document_switch", False):
+                    return
 
-                    size = os.path.getsize(file_path)
-                    if size > 0:
-                        break
-
-                except Exception:
-                    pass
-                time.sleep(0.1)
-            else:
+            gate_handle, error = self._acquire_file_scan_gate(file_path)
+            if not gate_handle:
+                if error in (ERROR_SHARING_VIOLATION, ERROR_LOCK_VIOLATION):
+                    self._queue_file_scan(file_path, FILE_SCAN_RETRY_SECONDS)
                 return
 
-            with self.lock_file_ops:
-                if file_path in self.virus_lock: return
+            try:
+                with self.lock_file_ops:
+                    if file_path in self.virus_lock:
+                        return
 
-            result = self.safe_scan_engine(file_path)
-            self.cloud_check(file_path)
-
-            if result:
-                if self.manage_named_list("quarantine", [file_path], action="add", lock_func=self.lock_file) > 0:
-                    self.write_log("BLOCK", "File Block", source=file_path, file_hash=self.calc_file_hash(file_path))
+                result = self.safe_scan_engine(file_path)
+                self.cloud_check(file_path)
 
+                if result:
+                    if self.manage_named_list("quarantine", [file_path], action="add", lock_func=self.lock_file) > 0:
+                        self.write_log("BLOCK", "File Block", source=file_path, file_hash=self.calc_file_hash(file_path))
+            finally:
+                self.kernel32.CloseHandle(gate_handle)
         except Exception:
             pass
 
@@ -1081,6 +1187,11 @@ class ProtectMixin:
                 self.advapi32.CloseServiceHandle(service)
             self.advapi32.CloseServiceHandle(scm)
 
+    def uninstall_system_driver(self):
+        if not self.stop_system_driver():
+            return False, 1051
+        return self._delete_driver_service()
+
     def _delete_driver_service(self):
         scm = self.advapi32.OpenSCManagerW(None, None, 0x0001)
         if not scm:

PYAS_Scanner.py

@@ -437,6 +437,47 @@ class ScannerMixin:
 
 ####################################################################################################
 
+    def _queue_virus_delete(self, file_path):
+        self._schedule_file_task("delete", file_path, self._retry_virus_delete, 0.5)
+
+    def _retry_virus_delete(self, file_path):
+        file_hash = self.calc_file_hash(file_path)
+        try:
+            with self.lock_file_ops:
+                if file_path in self.virus_lock:
+                    self.lock_file(file_path, False)
+
+            target_key = os.path.normcase(self.norm_path(file_path, must_exist=False) or file_path)
+            for proc in self.get_process_list():
+                proc_path = proc.get("path")
+                if not proc_path or proc_path == "None":
+                    continue
+                proc_key = os.path.normcase(self.norm_path(proc_path, must_exist=False) or proc_path)
+                if proc_key == target_key:
+                    self.kill_process(proc["pid"])
+
+            try:
+                os.chmod(file_path, stat.S_IWRITE)
+            except Exception:
+                pass
+            os.remove(file_path)
+        except FileNotFoundError:
+            pass
+        except Exception as e:
+            self.lock_file(file_path, True, quiet=True)
+            self._queue_virus_delete(file_path)
+            self.write_log("INFO", "Virus Delete Deferred", source=file_path, detail=str(e))
+            return
+
+        self.remove_list_items("quarantine", [file_path])
+        with self.lock_virus:
+            target_key = os.path.normcase(self.norm_path(file_path, must_exist=False) or file_path)
+            self.virus_results = [
+                result for result in self.virus_results
+                if os.path.normcase(self.norm_path(result, must_exist=False) or result) != target_key
+            ]
+        self.write_log("INFO", "Virus Delete", source=file_path, file_hash=file_hash, operate=True, success=True)
+
     def solve_scan(self, file_paths):
         deleted_paths = []
         proc_map = {}
@@ -477,23 +518,19 @@ class ScannerMixin:
                     except Exception:
                         pass
 
-                    delete_success = False
-                    for _ in range(5):
-                        try:
-                            os.remove(path)
-                            delete_success = True
-                            break
-
-                        except Exception:
-                            time.sleep(0.1)
-
-                    if not delete_success:
+                    file_hash = self.calc_file_hash(path)
+                    try:
                         os.remove(path)
+                    except Exception as e:
+                        self.lock_file(path, True, quiet=True)
+                        self._queue_virus_delete(path)
+                        self.write_log("INFO", "Virus Delete Deferred", source=path, detail=str(e))
+                        continue
 
                     self.remove_list_items("quarantine", [path])
-                    deleted_paths.append(raw_path) 
+                    deleted_paths.append(raw_path)
                     deleted_set.add(path)
-                    self.write_log("INFO", "Virus Delete", source=path, file_hash=self.calc_file_hash(path), operate=True, success=True)
+                    self.write_log("INFO", "Virus Delete", source=path, file_hash=file_hash, operate=True, success=True)
 
                 except Exception as e:
                     self.lock_file(path, True)

PYAS_Version.py

@@ -2,7 +2,7 @@ import pyinstaller_versionfile
 
 pyinstaller_versionfile.create_versionfile(
 output_file="versionfile.txt",
-version='3.6.3.0',
+version='3.6.4.0',
 company_name="PYAS Security",
 file_description="PYAS Security Antivirus",
 internal_name="PYAS",

Plugins/Rules/Rules_Driver_P1.json

@@ -70,9 +70,12 @@
                 "\\Device\\HarddiskVolume*\\Windows\\System32\\lsass.exe",
                 "\\Device\\HarddiskVolume*\\Windows\\System32\\services.exe",
                 "\\Device\\HarddiskVolume*\\Windows\\System32\\svchost.exe",
+                "\\Device\\HarddiskVolume*\\Windows\\System32\\msiexec.exe",
                 "\\Device\\HarddiskVolume*\\Windows\\SystemApps\\*\\SearchHost.exe",
                 "\\Device\\HarddiskVolume*\\Program Files\\*",
                 "\\Device\\HarddiskVolume*\\Program Files (x86)\\*",
+                "*\\games\\Arknights Game\\Arknights.exe",
+                "*\\games\\EndField Game\\EndField.exe",
             ],
             "Target": [
                 "*\\PYAS.exe",
@@ -280,6 +283,7 @@
                 "\\Device\\HarddiskVolume*\\Program Files (x86)\\*",
                 "\\Device\\HarddiskVolume*\\ProgramData\\Microsoft\\Windows Defender\\*",
                 "\\Device\\HarddiskVolume*\\ProgramData\\Microsoft\\Windows Defender Advanced Threat Protection\\*",
+                "\\Device\\HarddiskVolume*\\Users\\*\\AppData\\Local\\*",
                 "*\\Huorong\\Sysdiag\\bin\\HipsDaemon.exe",
                 "*\\360Safe.exe",
                 "*\\PYAS.exe",
@@ -547,6 +551,7 @@
                 "*\\ventoy\\Boot\\*",
                 "*\\AdminRegionConfig.*",
                 "*\\nt_qq\\nt_*\\*",
+                "\\Device\\HarddiskVolume*\\Users\\*\\Documents\\Tencent Files\\*\\nt_qq\\nt_*\\*",
             ],
             "Extensions": [
                 ".zip",
@@ -810,42 +815,19 @@
             "Priority": 170,
         },
         {
-            "Name": "Registry_Protect_Pending_File_Rename",
+            "Name": "Registry_Protect_OOBE_Reset",
             "Code": 3006,
             "Kill": false,
             "Category": "Registry",
-            "InitiatorExclude": [
-                "\\Device\\HarddiskVolume*\\Windows\\*",
-                "\\Device\\HarddiskVolume*\\ProgramData\\Microsoft\\Windows Defender\\*",
-                "\\Device\\HarddiskVolume*\\ProgramData\\Microsoft\\Windows Defender Advanced Threat Protection\\*",
-                "*\\Sources\\SetupHost.exe",
-                "*\\Work\\*\\DismHost.exe",
-                "*\\Huorong\\Sysdiag\\bin\\HipsDaemon.exe",
-                "*\\360Safe.exe",
-                "*\\PYAS.exe",
-            ],
-            "Target": [
-                "\\REGISTRY\\MACHINE\\SYSTEM\\*ControlSet*\\Control\\Session Manager\\PendingFileRenameOperations",
-                "\\REGISTRY\\MACHINE\\SYSTEM\\*ControlSet*\\Control\\Session Manager\\PendingFileRenameOperations2",
-            ],
-            "Operations": [
-                "Write",
-                "Delete",
-                "Create",
-            ],
-            "Priority": 140,
-        },
-        {
-            "Name": "Registry_Protect_OOBE_Reset",
-            "Code": 3007,
-            "Kill": true,
-            "Category": "Registry",
             "InitiatorExclude": [
                 "*\\Sources\\SetupHost.exe",
                 "*\\Work\\*\\DismHost.exe",
+                "*\\Windows\\UUS\\amd64\\MoUsoCoreWorker.exe",
+                "*\\Windows\\System32\\MoUsoCoreWorker.exe",
                 "*\\Windows\\System32\\Sysprep\\sysprep.exe",
                 "*\\Windows\\System32\\oobe\\*",
-                "*\\Windows\\UUS\\amd64\\MoUsoCoreWorker.exe",
+                "*\\Windows\\System32\\svchost.exe",
+                "*\\Windows\\explorer.exe",
                 "*\\PYAS.exe",
             ],
             "Target": [
@@ -862,6 +844,9 @@
                 "\\REGISTRY\\MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup\\OOBE\\*",
                 "\\REGISTRY\\MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\OOBE\\*",
             ],
+            "TargetExclude": [
+                "\\REGISTRY\\MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\OOBE\\TelemetryCorrelation",
+            ],
             "Operations": [
                 "Write",
                 "Delete",