Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c220035fd9 | ||
|
|
d7b98d3cde | ||
|
|
0ab876633f | ||
|
|
cabe76ca54 | ||
|
|
c57224e258 | ||
|
|
b56101d2e6 | ||
|
|
16211c28d1 | ||
|
|
139d3e0c3d | ||
|
|
f0652fc5ec | ||
|
|
55489cd634 | ||
|
|
742887c0e4 | ||
|
|
c3e5cd2c0d | ||
|
|
63e125627c | ||
|
|
c0189e9d05 |
No files matched your search
+144
-15
@@ -1,5 +1,8 @@
|
||||
import os, sys, math, time, datetime, pefile, hashlib
|
||||
import ctypes, ctypes.wintypes, multiprocessing
|
||||
import re, zlib
|
||||
|
||||
####################################################################################################
|
||||
|
||||
try:
|
||||
import orjson
|
||||
@@ -12,8 +15,6 @@ except ImportError:
|
||||
JSON_LOADS = json.loads
|
||||
HAS_ORJSON = False
|
||||
|
||||
####################################################################################################
|
||||
|
||||
JSONL_PATH = "pe_features.jsonl"
|
||||
BATCH_SIZE = 1000
|
||||
MAX_FILE_SIZE = 4 * 1024 * 1024 * 1024
|
||||
@@ -107,6 +108,8 @@ def verify_signature(file_path):
|
||||
####################################################################################################
|
||||
|
||||
class FeatureExtractor:
|
||||
_STRING_PATTERN = re.compile(b'[\x20-\x7E]{5,}')
|
||||
|
||||
@staticmethod
|
||||
def _safe_float(val):
|
||||
try:
|
||||
@@ -124,6 +127,86 @@ class FeatureExtractor:
|
||||
probs = counts[counts > 0] / len(arr)
|
||||
return float(-np.sum(probs * np.log2(probs)))
|
||||
|
||||
@classmethod
|
||||
def _extract_strings(cls, file_bytes):
|
||||
res = {"StringCount": 0.0, "StringMeanLength": 0.0, "StringEntropy": 0.0}
|
||||
for i in range(95):
|
||||
res[f'StringHist_{i:02d}'] = 0.0
|
||||
|
||||
count = 0
|
||||
total_len = 0
|
||||
char_counts = [0] * 95
|
||||
combined_data = bytearray()
|
||||
|
||||
for match in cls._STRING_PATTERN.finditer(file_bytes):
|
||||
m = match.group()
|
||||
count += 1
|
||||
length = len(m)
|
||||
total_len += length
|
||||
combined_data.extend(m)
|
||||
for b in m:
|
||||
char_counts[b - 0x20] += 1
|
||||
|
||||
if count == 0:
|
||||
return res
|
||||
|
||||
res["StringCount"] = float(count)
|
||||
res["StringMeanLength"] = float(total_len) / count
|
||||
res["StringEntropy"] = cls._calc_entropy(combined_data)
|
||||
|
||||
for i in range(95):
|
||||
res[f'StringHist_{i:02d}'] = float(char_counts[i]) / total_len
|
||||
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def _extract_histograms(cls, file_bytes):
|
||||
import numpy as np
|
||||
arr = np.frombuffer(file_bytes, dtype=np.uint8)
|
||||
sz = len(arr)
|
||||
res = {}
|
||||
|
||||
if sz == 0:
|
||||
for i in range(256):
|
||||
res[f'ByteHist_{i:02X}'] = 0.0
|
||||
res[f'ByteEnt_{i:02X}'] = 0.0
|
||||
return res
|
||||
|
||||
byte_counts = np.bincount(arr, minlength=256)
|
||||
byte_hist = byte_counts / sz
|
||||
for i, v in enumerate(byte_hist):
|
||||
res[f'ByteHist_{i:02X}'] = float(v)
|
||||
|
||||
ent_hist = np.zeros(256, dtype=np.float64)
|
||||
window = 2048
|
||||
step = 1024
|
||||
num_windows = (sz - window) // step + 1 if sz >= window else 1
|
||||
|
||||
for i in range(num_windows):
|
||||
w = arr if sz < window else arr[i*step : i*step+window]
|
||||
w_counts = np.bincount(w, minlength=256)
|
||||
|
||||
p = w_counts[w_counts > 0] / len(w)
|
||||
entropy = -np.sum(p * np.log2(p))
|
||||
|
||||
ent_bin = int(entropy * 2.0)
|
||||
if ent_bin > 15:
|
||||
ent_bin = 15
|
||||
|
||||
byte_bins = w_counts.reshape(16, 16).sum(axis=1)
|
||||
|
||||
idx_start = ent_bin * 16
|
||||
ent_hist[idx_start : idx_start+16] += byte_bins
|
||||
|
||||
ent_sum = np.sum(ent_hist)
|
||||
if ent_sum > 0:
|
||||
ent_hist /= ent_sum
|
||||
|
||||
for i, v in enumerate(ent_hist):
|
||||
res[f'ByteEnt_{i:02X}'] = float(v)
|
||||
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def _extract_overlay_features(cls, pe, fsize):
|
||||
overlay_offset = pe.get_overlay_data_start_offset()
|
||||
@@ -233,6 +316,42 @@ class FeatureExtractor:
|
||||
cfg["HasSEHTable"] = 1.0
|
||||
return cfg
|
||||
|
||||
@classmethod
|
||||
def _extract_security_directory(cls, pe, file_bytes, fsize):
|
||||
res = {"HasSignature": 0.0, "SignatureCount": 0.0}
|
||||
if not hasattr(pe, 'OPTIONAL_HEADER') or not hasattr(pe.OPTIONAL_HEADER, 'DATA_DIRECTORY'):
|
||||
return res
|
||||
|
||||
sec_idx = 4
|
||||
directories = pe.OPTIONAL_HEADER.DATA_DIRECTORY
|
||||
if len(directories) <= sec_idx:
|
||||
return res
|
||||
|
||||
sec_dir = directories[sec_idx]
|
||||
if sec_dir.Size == 0 or sec_dir.VirtualAddress == 0:
|
||||
return res
|
||||
|
||||
res["HasSignature"] = 1.0
|
||||
offset = sec_dir.VirtualAddress
|
||||
size = sec_dir.Size
|
||||
|
||||
if offset + size > fsize or offset < 0 or size < 0:
|
||||
return res
|
||||
|
||||
count = 0
|
||||
curr = offset
|
||||
end = offset + size
|
||||
|
||||
while curr + 8 <= end:
|
||||
dw_length = int.from_bytes(file_bytes[curr:curr+4], 'little')
|
||||
if dw_length < 8:
|
||||
break
|
||||
count += 1
|
||||
curr += (dw_length + 7) & ~7
|
||||
|
||||
res["SignatureCount"] = float(count)
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def extract(cls, file_bytes, file_path, fsize):
|
||||
if fsize == 0: return None
|
||||
@@ -241,6 +360,19 @@ class FeatureExtractor:
|
||||
try:
|
||||
pe = pefile.PE(data=file_bytes, fast_load=True)
|
||||
|
||||
try:
|
||||
pe.parse_rich_header()
|
||||
pe.parse_data_directories(directories=[
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_DEBUG'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_TLS'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']
|
||||
])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
base['TrustSigned'] = verify_signature(file_path)
|
||||
base['FileEntropy'] = cls._calc_entropy(file_bytes)
|
||||
base['FileSize'] = fsize
|
||||
@@ -283,12 +415,19 @@ class FeatureExtractor:
|
||||
base[f'Char_{flag:08X}_Count'] = 0.0
|
||||
base[f'Char_{flag:08X}_MeanEntropy'] = 0.0
|
||||
|
||||
for i in range(50):
|
||||
base[f'SectionHash_{i:02d}'] = 0.0
|
||||
|
||||
if hasattr(pe, 'sections'):
|
||||
base['SectionCount'] = len(pe.sections)
|
||||
entropies, raw_sizes, v_sizes = [], [], []
|
||||
exec_sec, write_sec, read_sec, sec_exc = 0, 0, 0, 0
|
||||
|
||||
for section in pe.sections:
|
||||
sec_name = section.Name.rstrip(b'\x00')
|
||||
hash_val = zlib.crc32(sec_name) % 50
|
||||
base[f'SectionHash_{hash_val:02d}'] += 1.0
|
||||
|
||||
try:
|
||||
s_data = section.get_data()
|
||||
s_entropy = cls._calc_entropy(s_data)
|
||||
@@ -345,17 +484,6 @@ class FeatureExtractor:
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
try:
|
||||
pe.parse_data_directories(directories=[
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_DEBUG'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_TLS'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']
|
||||
])
|
||||
except Exception: pass
|
||||
|
||||
if hasattr(pe, 'DIRECTORY_ENTRY_TLS') and hasattr(pe.DIRECTORY_ENTRY_TLS, 'struct'):
|
||||
base['HasTlsCallbacks'] = 1.0 if getattr(pe.DIRECTORY_ENTRY_TLS.struct, 'AddressOfCallBacks', 0) != 0 else 0.0
|
||||
|
||||
@@ -394,11 +522,14 @@ class FeatureExtractor:
|
||||
|
||||
base['DebugCount'] = len(pe.DIRECTORY_ENTRY_DEBUG) if hasattr(pe, 'DIRECTORY_ENTRY_DEBUG') else 0.0
|
||||
|
||||
base.update(cls._extract_strings(file_bytes))
|
||||
base.update(cls._extract_histograms(file_bytes))
|
||||
base.update(cls._extract_overlay_features(pe, fsize))
|
||||
base.update(cls._extract_rich_header(pe))
|
||||
base.update(cls._extract_ep_anomalies(pe))
|
||||
base.update(cls._extract_advanced_resources(pe))
|
||||
base.update(cls._extract_load_config(pe))
|
||||
base.update(cls._extract_security_directory(pe, file_bytes, fsize))
|
||||
|
||||
for k in base: base[k] = cls._safe_float(base[k])
|
||||
return {"Base": base, "DLLs": list(dlls), "APIs": list(apis)}
|
||||
@@ -420,8 +551,6 @@ def load_existing_hashes():
|
||||
except Exception: pass
|
||||
return hashes
|
||||
|
||||
####################################################################################################
|
||||
|
||||
GLOBAL_HASHES = set()
|
||||
|
||||
def worker_init(shared_hashes):
|
||||
|
||||
+162
-27
@@ -2,15 +2,13 @@ import os, sys, math, json, pefile, datetime
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
import ctypes, ctypes.wintypes
|
||||
import re, zlib
|
||||
|
||||
####################################################################################################
|
||||
|
||||
MODEL_FILE = "Pefile_General_S1.onnx"
|
||||
FEATURE_FILE = "features.json"
|
||||
MAX_FILE_SIZE = 4 * 1024 * 1024 * 1024
|
||||
TARGET_EXTENSIONS = {
|
||||
'.exe', '.dll', '.sys', '.ocx', '.scr', '.efi', '.acm', '.ax', '.cpl', '.drv', '.com', '.mui', '.pyd'
|
||||
}
|
||||
|
||||
####################################################################################################
|
||||
|
||||
@@ -101,6 +99,8 @@ class WinTrust:
|
||||
####################################################################################################
|
||||
|
||||
class FeatureExtractor:
|
||||
_STRING_PATTERN = re.compile(b'[\x20-\x7E]{5,}')
|
||||
|
||||
@staticmethod
|
||||
def _safe_float(val):
|
||||
try:
|
||||
@@ -118,6 +118,86 @@ class FeatureExtractor:
|
||||
probs = counts[counts > 0] / len(arr)
|
||||
return float(-np.sum(probs * np.log2(probs)))
|
||||
|
||||
@classmethod
|
||||
def _extract_strings(cls, file_bytes):
|
||||
res = {"StringCount": 0.0, "StringMeanLength": 0.0, "StringEntropy": 0.0}
|
||||
for i in range(95):
|
||||
res[f'StringHist_{i:02d}'] = 0.0
|
||||
|
||||
count = 0
|
||||
total_len = 0
|
||||
char_counts = [0] * 95
|
||||
combined_data = bytearray()
|
||||
|
||||
for match in cls._STRING_PATTERN.finditer(file_bytes):
|
||||
m = match.group()
|
||||
count += 1
|
||||
length = len(m)
|
||||
total_len += length
|
||||
combined_data.extend(m)
|
||||
for b in m:
|
||||
char_counts[b - 0x20] += 1
|
||||
|
||||
if count == 0:
|
||||
return res
|
||||
|
||||
res["StringCount"] = float(count)
|
||||
res["StringMeanLength"] = float(total_len) / count
|
||||
res["StringEntropy"] = cls._calc_entropy(combined_data)
|
||||
|
||||
for i in range(95):
|
||||
res[f'StringHist_{i:02d}'] = float(char_counts[i]) / total_len
|
||||
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def _extract_histograms(cls, file_bytes):
|
||||
import numpy as np
|
||||
arr = np.frombuffer(file_bytes, dtype=np.uint8)
|
||||
sz = len(arr)
|
||||
res = {}
|
||||
|
||||
if sz == 0:
|
||||
for i in range(256):
|
||||
res[f'ByteHist_{i:02X}'] = 0.0
|
||||
res[f'ByteEnt_{i:02X}'] = 0.0
|
||||
return res
|
||||
|
||||
byte_counts = np.bincount(arr, minlength=256)
|
||||
byte_hist = byte_counts / sz
|
||||
for i, v in enumerate(byte_hist):
|
||||
res[f'ByteHist_{i:02X}'] = float(v)
|
||||
|
||||
ent_hist = np.zeros(256, dtype=np.float64)
|
||||
window = 2048
|
||||
step = 1024
|
||||
num_windows = (sz - window) // step + 1 if sz >= window else 1
|
||||
|
||||
for i in range(num_windows):
|
||||
w = arr if sz < window else arr[i*step : i*step+window]
|
||||
w_counts = np.bincount(w, minlength=256)
|
||||
|
||||
p = w_counts[w_counts > 0] / len(w)
|
||||
entropy = -np.sum(p * np.log2(p))
|
||||
|
||||
ent_bin = int(entropy * 2.0)
|
||||
if ent_bin > 15:
|
||||
ent_bin = 15
|
||||
|
||||
byte_bins = w_counts.reshape(16, 16).sum(axis=1)
|
||||
|
||||
idx_start = ent_bin * 16
|
||||
ent_hist[idx_start : idx_start+16] += byte_bins
|
||||
|
||||
ent_sum = np.sum(ent_hist)
|
||||
if ent_sum > 0:
|
||||
ent_hist /= ent_sum
|
||||
|
||||
for i, v in enumerate(ent_hist):
|
||||
res[f'ByteEnt_{i:02X}'] = float(v)
|
||||
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def _extract_overlay_features(cls, pe, fsize):
|
||||
overlay_offset = pe.get_overlay_data_start_offset()
|
||||
@@ -227,6 +307,42 @@ class FeatureExtractor:
|
||||
cfg["HasSEHTable"] = 1.0
|
||||
return cfg
|
||||
|
||||
@classmethod
|
||||
def _extract_security_directory(cls, pe, file_bytes, fsize):
|
||||
res = {"HasSignature": 0.0, "SignatureCount": 0.0}
|
||||
if not hasattr(pe, 'OPTIONAL_HEADER') or not hasattr(pe.OPTIONAL_HEADER, 'DATA_DIRECTORY'):
|
||||
return res
|
||||
|
||||
sec_idx = 4
|
||||
directories = pe.OPTIONAL_HEADER.DATA_DIRECTORY
|
||||
if len(directories) <= sec_idx:
|
||||
return res
|
||||
|
||||
sec_dir = directories[sec_idx]
|
||||
if sec_dir.Size == 0 or sec_dir.VirtualAddress == 0:
|
||||
return res
|
||||
|
||||
res["HasSignature"] = 1.0
|
||||
offset = sec_dir.VirtualAddress
|
||||
size = sec_dir.Size
|
||||
|
||||
if offset + size > fsize or offset < 0 or size < 0:
|
||||
return res
|
||||
|
||||
count = 0
|
||||
curr = offset
|
||||
end = offset + size
|
||||
|
||||
while curr + 8 <= end:
|
||||
dw_length = int.from_bytes(file_bytes[curr:curr+4], 'little')
|
||||
if dw_length < 8:
|
||||
break
|
||||
count += 1
|
||||
curr += (dw_length + 7) & ~7
|
||||
|
||||
res["SignatureCount"] = float(count)
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def extract(cls, file_path):
|
||||
pe = None
|
||||
@@ -240,6 +356,19 @@ class FeatureExtractor:
|
||||
|
||||
base, dlls, apis = {}, set(), set()
|
||||
pe = pefile.PE(data=file_bytes, fast_load=True)
|
||||
|
||||
try:
|
||||
pe.parse_rich_header()
|
||||
pe.parse_data_directories(directories=[
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_DEBUG'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_TLS'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']
|
||||
])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
base['TrustSigned'] = float(WinTrust.verify(file_path))
|
||||
base['FileEntropy'] = cls._calc_entropy(file_bytes)
|
||||
@@ -283,12 +412,19 @@ class FeatureExtractor:
|
||||
base[f'Char_{flag:08X}_Count'] = 0.0
|
||||
base[f'Char_{flag:08X}_MeanEntropy'] = 0.0
|
||||
|
||||
for i in range(50):
|
||||
base[f'SectionHash_{i:02d}'] = 0.0
|
||||
|
||||
if hasattr(pe, 'sections'):
|
||||
base['SectionCount'] = len(pe.sections)
|
||||
entropies, raw_sizes, v_sizes = [], [], []
|
||||
exec_sec, write_sec, read_sec, sec_exc = 0, 0, 0, 0
|
||||
|
||||
for section in pe.sections:
|
||||
sec_name = section.Name.rstrip(b'\x00')
|
||||
hash_val = zlib.crc32(sec_name) % 50
|
||||
base[f'SectionHash_{hash_val:02d}'] += 1.0
|
||||
|
||||
try:
|
||||
s_data = section.get_data()
|
||||
s_entropy = cls._calc_entropy(s_data)
|
||||
@@ -345,17 +481,6 @@ class FeatureExtractor:
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
try:
|
||||
pe.parse_data_directories(directories=[
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_DEBUG'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_TLS'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']
|
||||
])
|
||||
except Exception: pass
|
||||
|
||||
if hasattr(pe, 'DIRECTORY_ENTRY_TLS') and hasattr(pe.DIRECTORY_ENTRY_TLS, 'struct'):
|
||||
base['HasTlsCallbacks'] = 1.0 if getattr(pe.DIRECTORY_ENTRY_TLS.struct, 'AddressOfCallBacks', 0) != 0 else 0.0
|
||||
|
||||
@@ -394,11 +519,14 @@ class FeatureExtractor:
|
||||
|
||||
base['DebugCount'] = len(pe.DIRECTORY_ENTRY_DEBUG) if hasattr(pe, 'DIRECTORY_ENTRY_DEBUG') else 0.0
|
||||
|
||||
base.update(cls._extract_strings(file_bytes))
|
||||
base.update(cls._extract_histograms(file_bytes))
|
||||
base.update(cls._extract_overlay_features(pe, fsize))
|
||||
base.update(cls._extract_rich_header(pe))
|
||||
base.update(cls._extract_ep_anomalies(pe))
|
||||
base.update(cls._extract_advanced_resources(pe))
|
||||
base.update(cls._extract_load_config(pe))
|
||||
base.update(cls._extract_security_directory(pe, file_bytes, fsize))
|
||||
|
||||
for k in base: base[k] = cls._safe_float(base[k])
|
||||
return {"Base": base, "DLLs": list(dlls), "APIs": list(apis)}
|
||||
@@ -420,6 +548,8 @@ class ModelPredictor:
|
||||
if current_dim != expected_dim:
|
||||
raise ValueError(f"Dimension mismatch! Model expects {expected_dim} features, but feature file lists {current_dim}.")
|
||||
|
||||
self.feat_map = {feat: i for i, feat in enumerate(self.model_features)}
|
||||
|
||||
def _load_features(self, path):
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(f"Feature file not found: {path}")
|
||||
@@ -429,18 +559,24 @@ class ModelPredictor:
|
||||
def predict(self, raw_data):
|
||||
vec = np.zeros((1, len(self.model_features)), dtype=np.float32)
|
||||
base = raw_data.get('Base', {})
|
||||
dlls = set(raw_data.get('DLLs', []))
|
||||
apis = set(raw_data.get('APIs', []))
|
||||
dlls = raw_data.get('DLLs', [])
|
||||
apis = raw_data.get('APIs', [])
|
||||
|
||||
for i, feat in enumerate(self.model_features):
|
||||
if feat.startswith('Dll_'):
|
||||
if feat[4:] in dlls:
|
||||
vec[0, i] = 1.0
|
||||
elif feat.startswith('Api_'):
|
||||
if feat[4:] in apis:
|
||||
vec[0, i] = 1.0
|
||||
else:
|
||||
vec[0, i] = base.get(feat, 0.0)
|
||||
for k, v in base.items():
|
||||
if k in self.feat_map:
|
||||
vec[0, self.feat_map[k]] = v
|
||||
|
||||
for d in dlls:
|
||||
h = zlib.crc32(d.encode('utf-8', 'ignore')) % 256
|
||||
feat_name = f"DllHash_{h:03d}"
|
||||
if feat_name in self.feat_map:
|
||||
vec[0, self.feat_map[feat_name]] += 1.0
|
||||
|
||||
for a in apis:
|
||||
h = zlib.crc32(a.encode('utf-8', 'ignore')) % 1024
|
||||
feat_name = f"ApiHash_{h:04d}"
|
||||
if feat_name in self.feat_map:
|
||||
vec[0, self.feat_map[feat_name]] += 1.0
|
||||
|
||||
outputs = self.sess.run(None, {self.input_name: vec})
|
||||
|
||||
@@ -464,8 +600,7 @@ def scan_target(target, predictor):
|
||||
elif os.path.isdir(target):
|
||||
for r, _, fs in os.walk(target):
|
||||
for f in fs:
|
||||
if os.path.splitext(f)[1].lower() in TARGET_EXTENSIONS:
|
||||
files.append(os.path.join(r, f))
|
||||
files.append(os.path.join(r, f))
|
||||
|
||||
if not files:
|
||||
print("[-] No valid PE files found.")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import os, sys, time, json, onnxmltools
|
||||
import os, sys, time, json, onnxmltools, zlib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import lightgbm as lgb
|
||||
@@ -9,6 +9,12 @@ from onnxmltools.convert.common.data_types import FloatTensorType
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import accuracy_score, roc_auc_score
|
||||
|
||||
try:
|
||||
import orjson
|
||||
JSON_LOADS = orjson.loads
|
||||
except ImportError:
|
||||
JSON_LOADS = json.loads
|
||||
|
||||
####################################################################################################
|
||||
|
||||
JSONL_PATH = "pe_features.jsonl"
|
||||
@@ -18,6 +24,9 @@ FEATURE_FILE = "features.json"
|
||||
TEST_SIZE = 0.0001
|
||||
RANDOM_SEED = 42
|
||||
|
||||
DLL_HASH_DIM = 256
|
||||
API_HASH_DIM = 1024
|
||||
|
||||
EXCLUDE_FEATURES = {
|
||||
'label', 'filehash',
|
||||
}
|
||||
@@ -42,26 +51,22 @@ LGBM_PARAMS = {
|
||||
def analyze_schema(jsonl_path):
|
||||
print("[*] Stage 1: Scanning global schema...")
|
||||
base_keys = set()
|
||||
dll_counts = defaultdict(int)
|
||||
api_counts = defaultdict(int)
|
||||
num_lines = 0
|
||||
|
||||
with open(jsonl_path, 'r') as f:
|
||||
with open(jsonl_path, 'rb') as f:
|
||||
for line in f:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
base_keys.update(data.get('Base', {}).keys())
|
||||
for d in data.get('DLLs', []):
|
||||
dll_counts[d] += 1
|
||||
for a in data.get('APIs', []):
|
||||
api_counts[a] += 1
|
||||
data = JSON_LOADS(line)
|
||||
base_data = data.get('Base')
|
||||
if base_data:
|
||||
base_keys.update(base_data.keys())
|
||||
num_lines += 1
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
base_schema = sorted([k for k in base_keys if k.lower().strip() not in EXCLUDE_FEATURES])
|
||||
dll_schema = sorted([f"Dll_{k}" for k, v in dll_counts.items() if v >= 1000])
|
||||
api_schema = sorted([f"Api_{k}" for k, v in api_counts.items() if v >= 1000])
|
||||
dll_schema = [f"DllHash_{i:03d}" for i in range(DLL_HASH_DIM)]
|
||||
api_schema = [f"ApiHash_{i:04d}" for i in range(API_HASH_DIM)]
|
||||
|
||||
return base_schema, dll_schema, api_schema, num_lines
|
||||
|
||||
@@ -71,39 +76,46 @@ def load_data_efficiently(jsonl_path, base_schema, dll_schema, api_schema, num_l
|
||||
print(f"[*] Stage 2: Building sparse feature matrix dynamically (Expected samples: {num_lines})...")
|
||||
final_features = base_schema + dll_schema + api_schema
|
||||
num_features = len(final_features)
|
||||
feature_idx = {feat: i for i, feat in enumerate(final_features)}
|
||||
|
||||
base_idx = {feat: i for i, feat in enumerate(base_schema)}
|
||||
offset_dll = len(base_schema)
|
||||
offset_api = offset_dll + DLL_HASH_DIM
|
||||
|
||||
rows, cols, vals = [], [], []
|
||||
y = np.zeros(num_lines, dtype=np.int8)
|
||||
|
||||
with open(jsonl_path, 'r') as f:
|
||||
with open(jsonl_path, 'rb') as f:
|
||||
row_idx = 0
|
||||
for line in f:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
data = JSON_LOADS(line)
|
||||
y[row_idx] = data.get('Label', 0)
|
||||
|
||||
for k, v in data.get('Base', {}).items():
|
||||
idx = feature_idx.get(k)
|
||||
if idx is not None:
|
||||
val = float(v)
|
||||
if val != 0.0:
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
vals.append(val)
|
||||
base_data = data.get('Base')
|
||||
if base_data:
|
||||
for k, v in base_data.items():
|
||||
idx = base_idx.get(k)
|
||||
if idx is not None:
|
||||
val = float(v)
|
||||
if val != 0.0:
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
vals.append(val)
|
||||
|
||||
for d in data.get('DLLs', []):
|
||||
idx = feature_idx.get(f"Dll_{d}")
|
||||
if idx is not None:
|
||||
dll_data = data.get('DLLs')
|
||||
if dll_data:
|
||||
for d in dll_data:
|
||||
idx = zlib.crc32(d.encode('utf-8', 'ignore')) % DLL_HASH_DIM
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
cols.append(offset_dll + idx)
|
||||
vals.append(1.0)
|
||||
|
||||
for a in data.get('APIs', []):
|
||||
idx = feature_idx.get(f"Api_{a}")
|
||||
if idx is not None:
|
||||
api_data = data.get('APIs')
|
||||
if api_data:
|
||||
for a in api_data:
|
||||
idx = zlib.crc32(a.encode('utf-8', 'ignore')) % API_HASH_DIM
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
cols.append(offset_api + idx)
|
||||
vals.append(1.0)
|
||||
|
||||
row_idx += 1
|
||||
@@ -131,7 +143,7 @@ def train_process(X, y, feature_names):
|
||||
n_neg = (y_train == 0).sum()
|
||||
|
||||
base_weight = n_neg / n_pos if n_pos > 0 else 1.0
|
||||
weight_ratio_target = 0.01
|
||||
weight_ratio_target = 0.005
|
||||
final_pos_weight = base_weight * weight_ratio_target
|
||||
|
||||
print(f"[*] Sample distribution: Safe={n_neg}, Malware={n_pos}")
|
||||
@@ -233,8 +245,8 @@ if __name__ == "__main__":
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[*] Detected Base features: {len(base_schema)}")
|
||||
print(f"[*] Filtered DLL features: {len(dll_schema)}")
|
||||
print(f"[*] Filtered API features: {len(api_schema)}")
|
||||
print(f"[*] Hashing DLL features: {len(dll_schema)}")
|
||||
print(f"[*] Hashing API features: {len(api_schema)}")
|
||||
|
||||
X, y, final_features = load_data_efficiently(JSONL_PATH, base_schema, dll_schema, api_schema, num_lines)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import os, sys, time, json, onnxmltools
|
||||
import os, sys, time, json, onnxmltools, zlib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import lightgbm as lgb
|
||||
@@ -9,6 +9,12 @@ from onnxmltools.convert.common.data_types import FloatTensorType
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.metrics import accuracy_score, roc_auc_score
|
||||
|
||||
try:
|
||||
import orjson
|
||||
JSON_LOADS = orjson.loads
|
||||
except ImportError:
|
||||
JSON_LOADS = json.loads
|
||||
|
||||
####################################################################################################
|
||||
|
||||
JSONL_PATH = "pe_features.jsonl"
|
||||
@@ -18,6 +24,9 @@ FEATURE_FILE = "features.json"
|
||||
TEST_SIZE = 0.0001
|
||||
RANDOM_SEED = 42
|
||||
|
||||
DLL_HASH_DIM = 256
|
||||
API_HASH_DIM = 1024
|
||||
|
||||
EXCLUDE_FEATURES = {
|
||||
'label', 'filehash',
|
||||
}
|
||||
@@ -42,26 +51,22 @@ LGBM_PARAMS = {
|
||||
def analyze_schema(jsonl_path):
|
||||
print("[*] Stage 1: Scanning global schema...")
|
||||
base_keys = set()
|
||||
dll_counts = defaultdict(int)
|
||||
api_counts = defaultdict(int)
|
||||
num_lines = 0
|
||||
|
||||
with open(jsonl_path, 'r') as f:
|
||||
with open(jsonl_path, 'rb') as f:
|
||||
for line in f:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
base_keys.update(data.get('Base', {}).keys())
|
||||
for d in data.get('DLLs', []):
|
||||
dll_counts[d] += 1
|
||||
for a in data.get('APIs', []):
|
||||
api_counts[a] += 1
|
||||
data = JSON_LOADS(line)
|
||||
base_data = data.get('Base')
|
||||
if base_data:
|
||||
base_keys.update(base_data.keys())
|
||||
num_lines += 1
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
base_schema = sorted([k for k in base_keys if k.lower().strip() not in EXCLUDE_FEATURES])
|
||||
dll_schema = sorted([f"Dll_{k}" for k, v in dll_counts.items() if v >= 1000])
|
||||
api_schema = sorted([f"Api_{k}" for k, v in api_counts.items() if v >= 1000])
|
||||
dll_schema = [f"DllHash_{i:03d}" for i in range(DLL_HASH_DIM)]
|
||||
api_schema = [f"ApiHash_{i:04d}" for i in range(API_HASH_DIM)]
|
||||
|
||||
return base_schema, dll_schema, api_schema, num_lines
|
||||
|
||||
@@ -71,39 +76,46 @@ def load_data_efficiently(jsonl_path, base_schema, dll_schema, api_schema, num_l
|
||||
print(f"[*] Stage 2: Building sparse feature matrix dynamically (Expected samples: {num_lines})...")
|
||||
final_features = base_schema + dll_schema + api_schema
|
||||
num_features = len(final_features)
|
||||
feature_idx = {feat: i for i, feat in enumerate(final_features)}
|
||||
|
||||
base_idx = {feat: i for i, feat in enumerate(base_schema)}
|
||||
offset_dll = len(base_schema)
|
||||
offset_api = offset_dll + DLL_HASH_DIM
|
||||
|
||||
rows, cols, vals = [], [], []
|
||||
y = np.zeros(num_lines, dtype=np.int8)
|
||||
|
||||
with open(jsonl_path, 'r') as f:
|
||||
with open(jsonl_path, 'rb') as f:
|
||||
row_idx = 0
|
||||
for line in f:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
data = JSON_LOADS(line)
|
||||
y[row_idx] = data.get('Label', 0)
|
||||
|
||||
for k, v in data.get('Base', {}).items():
|
||||
idx = feature_idx.get(k)
|
||||
if idx is not None:
|
||||
val = float(v)
|
||||
if val != 0.0:
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
vals.append(val)
|
||||
base_data = data.get('Base')
|
||||
if base_data:
|
||||
for k, v in base_data.items():
|
||||
idx = base_idx.get(k)
|
||||
if idx is not None:
|
||||
val = float(v)
|
||||
if val != 0.0:
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
vals.append(val)
|
||||
|
||||
for d in data.get('DLLs', []):
|
||||
idx = feature_idx.get(f"Dll_{d}")
|
||||
if idx is not None:
|
||||
dll_data = data.get('DLLs')
|
||||
if dll_data:
|
||||
for d in dll_data:
|
||||
idx = zlib.crc32(d.encode('utf-8', 'ignore')) % DLL_HASH_DIM
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
cols.append(offset_dll + idx)
|
||||
vals.append(1.0)
|
||||
|
||||
for a in data.get('APIs', []):
|
||||
idx = feature_idx.get(f"Api_{a}")
|
||||
if idx is not None:
|
||||
api_data = data.get('APIs')
|
||||
if api_data:
|
||||
for a in api_data:
|
||||
idx = zlib.crc32(a.encode('utf-8', 'ignore')) % API_HASH_DIM
|
||||
rows.append(row_idx)
|
||||
cols.append(idx)
|
||||
cols.append(offset_api + idx)
|
||||
vals.append(1.0)
|
||||
|
||||
row_idx += 1
|
||||
@@ -233,8 +245,8 @@ if __name__ == "__main__":
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[*] Detected Base features: {len(base_schema)}")
|
||||
print(f"[*] Filtered DLL features: {len(dll_schema)}")
|
||||
print(f"[*] Filtered API features: {len(api_schema)}")
|
||||
print(f"[*] Hashing DLL features: {len(dll_schema)}")
|
||||
print(f"[*] Hashing API features: {len(api_schema)}")
|
||||
|
||||
X, y, final_features = load_data_efficiently(JSONL_PATH, base_schema, dll_schema, api_schema, num_lines)
|
||||
|
||||
|
||||
@@ -721,7 +721,14 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
});
|
||||
|
||||
['whitelist_window', 'quarantine_window', 'popup_window', 'custom_protect_window'].forEach(id => {
|
||||
const listKey = id === 'popup_window' ? 'block_list' : (id === 'custom_protect_window' ? 'custom_rule' : id.replace('_window', ''));
|
||||
const listKeyMap = {
|
||||
'popup_window': 'block_list',
|
||||
'custom_protect_window': 'custom_rule',
|
||||
'whitelist_window': 'white_list',
|
||||
'quarantine_window': 'quarantine'
|
||||
};
|
||||
const listKey = listKeyMap[id];
|
||||
|
||||
document.querySelector(`#${id} .modern-btn:not(.primary-btn)`)?.addEventListener('click', () => {
|
||||
const checked = getCheckedValues(`#${id}`);
|
||||
if (checked.length > 0 && window.pywebview) window.pywebview.api.remove_list_items(listKey, checked).then(refreshConfigLists);
|
||||
|
||||
+208
-60
@@ -1,4 +1,4 @@
|
||||
import os, re, yara, time, math, json, numpy, base64, datetime, requests
|
||||
import os, re, yara, time, math, json, numpy, base64, datetime, requests, zlib
|
||||
import ctypes, ctypes.wintypes, hashlib, pefile, threading, onnxruntime
|
||||
|
||||
####################################################################################################
|
||||
@@ -154,7 +154,22 @@ class rule_scanner:
|
||||
rule_name = str(matches[0])
|
||||
label = rule_name.split("_")[0]
|
||||
level = rule_name.split("_")[-1]
|
||||
return f"{label}:WinPE/Unknown.A!rb", level
|
||||
return f"{label}:WinPE/Unknown.{level}!yr", level
|
||||
return False, False
|
||||
except Exception:
|
||||
return False, False
|
||||
|
||||
def yara_mem_scan(self, pid):
|
||||
try:
|
||||
if not self.rules:
|
||||
return False, False
|
||||
|
||||
matches = self.rules.match(pid=pid)
|
||||
if matches:
|
||||
rule_name = str(matches[0])
|
||||
label = rule_name.split("_")[0]
|
||||
level = rule_name.split("_")[-1]
|
||||
return f"{label}:WinPE/Unknown.{level}!ym", level
|
||||
return False, False
|
||||
except Exception:
|
||||
return False, False
|
||||
@@ -162,6 +177,8 @@ class rule_scanner:
|
||||
####################################################################################################
|
||||
|
||||
class pe_scanner:
|
||||
_STRING_PATTERN = re.compile(b'[\x20-\x7E]{5,}')
|
||||
|
||||
def __init__(self):
|
||||
self.model = None
|
||||
self.input_name = None
|
||||
@@ -208,6 +225,82 @@ class pe_scanner:
|
||||
probs = counts[counts > 0] / len(arr)
|
||||
return float(-numpy.sum(probs * numpy.log2(probs)))
|
||||
|
||||
def _extract_strings(self, file_bytes):
|
||||
res = {"StringCount": 0.0, "StringMeanLength": 0.0, "StringEntropy": 0.0}
|
||||
for i in range(95):
|
||||
res[f'StringHist_{i:02d}'] = 0.0
|
||||
|
||||
count = 0
|
||||
total_len = 0
|
||||
char_counts = [0] * 95
|
||||
combined_data = bytearray()
|
||||
|
||||
for match in self._STRING_PATTERN.finditer(file_bytes):
|
||||
m = match.group()
|
||||
count += 1
|
||||
length = len(m)
|
||||
total_len += length
|
||||
combined_data.extend(m)
|
||||
for b in m:
|
||||
char_counts[b - 0x20] += 1
|
||||
|
||||
if count == 0:
|
||||
return res
|
||||
|
||||
res["StringCount"] = float(count)
|
||||
res["StringMeanLength"] = float(total_len) / count
|
||||
res["StringEntropy"] = self._calc_entropy(combined_data)
|
||||
|
||||
for i in range(95):
|
||||
res[f'StringHist_{i:02d}'] = float(char_counts[i]) / total_len
|
||||
|
||||
return res
|
||||
|
||||
def _extract_histograms(self, file_bytes):
|
||||
arr = numpy.frombuffer(file_bytes, dtype=numpy.uint8)
|
||||
sz = len(arr)
|
||||
res = {}
|
||||
|
||||
if sz == 0:
|
||||
for i in range(256):
|
||||
res[f'ByteHist_{i:02X}'] = 0.0
|
||||
res[f'ByteEnt_{i:02X}'] = 0.0
|
||||
return res
|
||||
|
||||
byte_counts = numpy.bincount(arr, minlength=256)
|
||||
byte_hist = byte_counts / sz
|
||||
for i, v in enumerate(byte_hist):
|
||||
res[f'ByteHist_{i:02X}'] = float(v)
|
||||
|
||||
ent_hist = numpy.zeros(256, dtype=numpy.float64)
|
||||
window = 2048
|
||||
step = 1024
|
||||
num_windows = (sz - window) // step + 1 if sz >= window else 1
|
||||
|
||||
for i in range(num_windows):
|
||||
w = arr if sz < window else arr[i*step : i*step+window]
|
||||
w_counts = numpy.bincount(w, minlength=256)
|
||||
|
||||
p = w_counts[w_counts > 0] / len(w)
|
||||
entropy = -numpy.sum(p * numpy.log2(p))
|
||||
|
||||
ent_bin = int(entropy * 2.0)
|
||||
if ent_bin > 15:
|
||||
ent_bin = 15
|
||||
|
||||
byte_bins = w_counts.reshape(16, 16).sum(axis=1)
|
||||
idx_start = ent_bin * 16
|
||||
ent_hist[idx_start : idx_start+16] += byte_bins
|
||||
|
||||
ent_sum = numpy.sum(ent_hist)
|
||||
if ent_sum > 0:
|
||||
ent_hist /= ent_sum
|
||||
|
||||
for i, v in enumerate(ent_hist):
|
||||
res[f'ByteEnt_{i:02X}'] = float(v)
|
||||
|
||||
return res
|
||||
|
||||
def _extract_overlay_features(self, pe, fsize):
|
||||
overlay_offset = pe.get_overlay_data_start_offset()
|
||||
if not overlay_offset or overlay_offset >= fsize:
|
||||
@@ -312,6 +405,41 @@ class pe_scanner:
|
||||
cfg["HasSEHTable"] = 1.0
|
||||
return cfg
|
||||
|
||||
def _extract_security_directory(self, pe, file_bytes, fsize):
|
||||
res = {"HasSignature": 0.0, "SignatureCount": 0.0}
|
||||
if not hasattr(pe, 'OPTIONAL_HEADER') or not hasattr(pe.OPTIONAL_HEADER, 'DATA_DIRECTORY'):
|
||||
return res
|
||||
|
||||
sec_idx = 4
|
||||
directories = pe.OPTIONAL_HEADER.DATA_DIRECTORY
|
||||
if len(directories) <= sec_idx:
|
||||
return res
|
||||
|
||||
sec_dir = directories[sec_idx]
|
||||
if sec_dir.Size == 0 or sec_dir.VirtualAddress == 0:
|
||||
return res
|
||||
|
||||
res["HasSignature"] = 1.0
|
||||
offset = sec_dir.VirtualAddress
|
||||
size = sec_dir.Size
|
||||
|
||||
if offset + size > fsize or offset < 0 or size < 0:
|
||||
return res
|
||||
|
||||
count = 0
|
||||
curr = offset
|
||||
end = offset + size
|
||||
|
||||
while curr + 8 <= end:
|
||||
dw_length = int.from_bytes(file_bytes[curr:curr+4], 'little')
|
||||
if dw_length < 8:
|
||||
break
|
||||
count += 1
|
||||
curr += (dw_length + 7) & ~7
|
||||
|
||||
res["SignatureCount"] = float(count)
|
||||
return res
|
||||
|
||||
def extract_features(self, file_path):
|
||||
pe = None
|
||||
try:
|
||||
@@ -327,6 +455,19 @@ class pe_scanner:
|
||||
apis = set()
|
||||
pe = pefile.PE(data=file_bytes, fast_load=True)
|
||||
|
||||
try:
|
||||
pe.parse_rich_header()
|
||||
pe.parse_data_directories(directories=[
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_DEBUG'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_TLS'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']
|
||||
])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
base['TrustSigned'] = 1.0 if self.signer.sign_verify(file_path) else 0.0
|
||||
base['FileEntropy'] = self._calc_entropy(file_bytes)
|
||||
base['FileSize'] = float(fsize)
|
||||
@@ -380,6 +521,9 @@ class pe_scanner:
|
||||
base[f'Char_{flag:08X}_Count'] = 0.0
|
||||
base[f'Char_{flag:08X}_MeanEntropy'] = 0.0
|
||||
|
||||
for i in range(50):
|
||||
base[f'SectionHash_{i:02d}'] = 0.0
|
||||
|
||||
if hasattr(pe, 'sections'):
|
||||
base['SectionCount'] = len(pe.sections)
|
||||
entropies = []
|
||||
@@ -391,6 +535,10 @@ class pe_scanner:
|
||||
sec_exc = 0
|
||||
|
||||
for section in pe.sections:
|
||||
sec_name = section.Name.rstrip(b'\x00')
|
||||
hash_val = zlib.crc32(sec_name) % 50
|
||||
base[f'SectionHash_{hash_val:02d}'] += 1.0
|
||||
|
||||
try:
|
||||
s_data = section.get_data()
|
||||
s_entropy = self._calc_entropy(s_data)
|
||||
@@ -447,17 +595,6 @@ class pe_scanner:
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
try:
|
||||
pe.parse_data_directories(directories=[
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_DEBUG'],
|
||||
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_TLS']
|
||||
])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if hasattr(pe, 'DIRECTORY_ENTRY_TLS') and hasattr(pe.DIRECTORY_ENTRY_TLS, 'struct'):
|
||||
if getattr(pe.DIRECTORY_ENTRY_TLS.struct, 'AddressOfCallBacks', 0) != 0:
|
||||
base['HasTlsCallbacks'] = 1.0
|
||||
@@ -510,11 +647,14 @@ class pe_scanner:
|
||||
else:
|
||||
base['DebugCount'] = 0.0
|
||||
|
||||
base.update(self._extract_strings(file_bytes))
|
||||
base.update(self._extract_histograms(file_bytes))
|
||||
base.update(self._extract_overlay_features(pe, fsize))
|
||||
base.update(self._extract_rich_header(pe))
|
||||
base.update(self._extract_ep_anomalies(pe))
|
||||
base.update(self._extract_advanced_resources(pe))
|
||||
base.update(self._extract_load_config(pe))
|
||||
base.update(self._extract_security_directory(pe, file_bytes, fsize))
|
||||
|
||||
for k in base:
|
||||
base[k] = self._safe_float(base[k])
|
||||
@@ -539,16 +679,29 @@ class pe_scanner:
|
||||
base = raw_data.get('Base', {})
|
||||
dlls = set(raw_data.get('DLLs', []))
|
||||
apis = set(raw_data.get('APIs', []))
|
||||
feat_map = {feat: i for i, feat in enumerate(self.feature_order)}
|
||||
|
||||
for i, feat in enumerate(self.feature_order):
|
||||
if feat.startswith('Dll_'):
|
||||
if feat[4:] in dlls:
|
||||
vec[0, i] = 1.0
|
||||
elif feat.startswith('Api_'):
|
||||
if feat[4:] in apis:
|
||||
vec[0, i] = 1.0
|
||||
else:
|
||||
vec[0, i] = base.get(feat, 0.0)
|
||||
for k, v in base.items():
|
||||
if k in feat_map:
|
||||
vec[0, feat_map[k]] = v
|
||||
|
||||
for d in dlls:
|
||||
old_fn = f"Dll_{d}"
|
||||
if old_fn in feat_map:
|
||||
vec[0, feat_map[old_fn]] = 1.0
|
||||
h = zlib.crc32(d.encode('utf-8', 'ignore')) % 256
|
||||
new_fn = f"DllHash_{h:03d}"
|
||||
if new_fn in feat_map:
|
||||
vec[0, feat_map[new_fn]] += 1.0
|
||||
|
||||
for a in apis:
|
||||
old_fn = f"Api_{a}"
|
||||
if old_fn in feat_map:
|
||||
vec[0, feat_map[old_fn]] = 1.0
|
||||
h = zlib.crc32(a.encode('utf-8', 'ignore')) % 1024
|
||||
new_fn = f"ApiHash_{h:04d}"
|
||||
if new_fn in feat_map:
|
||||
vec[0, feat_map[new_fn]] += 1.0
|
||||
|
||||
outputs = self.model.run(None, {self.input_name: vec})
|
||||
result = outputs[1]
|
||||
@@ -624,7 +777,7 @@ class cloud_scanner:
|
||||
r = self._request("POST", f"/api/rescan/{sha256}")
|
||||
return r is not None and r.json().get('status') == 'success'
|
||||
|
||||
def upload_file(self, file_path, api_host, api_key, chunk_size=5242880, need_rescan=False):
|
||||
def upload_file(self, file_path, api_host, api_key, chunk_size=4194304, need_rescan=False, max_retries=3):
|
||||
try:
|
||||
self._init_session(api_host, api_key)
|
||||
sha256 = self.calc_hash(file_path)
|
||||
@@ -633,9 +786,7 @@ class cloud_scanner:
|
||||
|
||||
status_req = self._request("GET", f"/api/processing_status/{sha256}")
|
||||
if status_req:
|
||||
status_data = status_req.json()
|
||||
current_status = status_data.get('status')
|
||||
|
||||
current_status = status_req.json().get('status')
|
||||
if current_status == 'done':
|
||||
if need_rescan:
|
||||
self.rescan(sha256, api_host, api_key)
|
||||
@@ -647,40 +798,37 @@ class cloud_scanner:
|
||||
if file_size > 104857600:
|
||||
return False, sha256
|
||||
|
||||
if file_size > chunk_size:
|
||||
total_chunks = (file_size + chunk_size - 1) // chunk_size
|
||||
upload_id = os.urandom(16).hex()
|
||||
with open(file_path, 'rb') as f:
|
||||
for i in range(total_chunks):
|
||||
chunk_data = f.read(chunk_size)
|
||||
headers = {
|
||||
"X-Chunk-Index": str(i),
|
||||
"X-Total-Chunks": str(total_chunks),
|
||||
"X-Upload-ID": upload_id
|
||||
}
|
||||
|
||||
chunk_success = False
|
||||
for _ in range(3):
|
||||
r = self._request("POST", "/api/upload", files={'file': (os.path.basename(file_path), chunk_data)}, headers=headers)
|
||||
if r:
|
||||
chunk_success = True
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
if not chunk_success:
|
||||
return False, sha256
|
||||
return True, sha256
|
||||
|
||||
total_chunks = max(1, (file_size + chunk_size - 1) // chunk_size)
|
||||
upload_id = os.urandom(16).hex()
|
||||
|
||||
with open(file_path, 'rb') as f:
|
||||
file_data = f.read()
|
||||
|
||||
for _ in range(3):
|
||||
r = self._request("POST", "/api/upload", files={'file': (os.path.basename(file_path), file_data)})
|
||||
if r:
|
||||
return True, sha256
|
||||
time.sleep(1)
|
||||
|
||||
return False, sha256
|
||||
for i in range(total_chunks):
|
||||
chunk_data = f.read(chunk_size)
|
||||
headers = {
|
||||
"X-Chunk-Index": str(i),
|
||||
"X-Total-Chunks": str(total_chunks),
|
||||
"X-Upload-ID": upload_id
|
||||
}
|
||||
|
||||
chunk_success = False
|
||||
for attempt in range(max_retries):
|
||||
r = self._request("POST", "/api/upload", files={'file': (os.path.basename(file_path), chunk_data)}, headers=headers)
|
||||
if r:
|
||||
if i == total_chunks - 1:
|
||||
try:
|
||||
resp = r.json()
|
||||
if 'url' in resp:
|
||||
sha256 = resp.get('url', '').split('/')[-1]
|
||||
except Exception:
|
||||
pass
|
||||
chunk_success = True
|
||||
break
|
||||
time.sleep(2 ** attempt)
|
||||
|
||||
if not chunk_success:
|
||||
return False, sha256
|
||||
|
||||
return True, sha256
|
||||
except Exception:
|
||||
return False, None
|
||||
|
||||
@@ -725,7 +873,7 @@ class cloud_scanner:
|
||||
sim_malicious_count += 1
|
||||
|
||||
if is_malicious and (valid_sim_count == 0 or sim_malicious_count == valid_sim_count):
|
||||
return f"General:WinPE/Unknown.{score}!cl"
|
||||
return f"General:WinPE/Malware.{score}!cl"
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
+1
-1
@@ -2,7 +2,7 @@ import pyinstaller_versionfile
|
||||
|
||||
pyinstaller_versionfile.create_versionfile(
|
||||
output_file="versionfile.txt",
|
||||
version='3.5.5.0',
|
||||
version='3.5.6.0',
|
||||
company_name="PYAS Security",
|
||||
file_description="PYAS Security Antivirus",
|
||||
internal_name="PYAS",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "DriverCommon.h"
|
||||
|
||||
DRIVER_DATA GlobalData;
|
||||
static BOOLEAN g_ImageNotifyRegistered = FALSE;
|
||||
|
||||
static NTSTATUS InstanceSetup(PCFLT_RELATED_OBJECTS FltObjects, FLT_INSTANCE_SETUP_FLAGS Flags, DEVICE_TYPE VolumeDeviceType, FLT_FILESYSTEM_TYPE VolumeFilesystemType) {
|
||||
UNREFERENCED_PARAMETER(FltObjects);
|
||||
@@ -27,9 +28,7 @@ static VOID InstanceTeardownComplete(PCFLT_RELATED_OBJECTS FltObjects, FLT_INSTA
|
||||
}
|
||||
|
||||
static NTSTATUS DriverUnload(FLT_FILTER_UNLOAD_FLAGS Flags) {
|
||||
if (!(Flags & FLTFL_FILTER_UNLOAD_MANDATORY)) {
|
||||
return STATUS_FLT_DO_NOT_DETACH;
|
||||
}
|
||||
UNREFERENCED_PARAMETER(Flags);
|
||||
|
||||
if (GlobalData.ServerPort) {
|
||||
FltCloseCommunicationPort(GlobalData.ServerPort);
|
||||
@@ -41,12 +40,14 @@ static NTSTATUS DriverUnload(FLT_FILTER_UNLOAD_FLAGS Flags) {
|
||||
GlobalData.FilterHandle = NULL;
|
||||
}
|
||||
|
||||
PsRemoveLoadImageNotifyRoutine(ImageLoadNotify);
|
||||
if (g_ImageNotifyRegistered) {
|
||||
PsRemoveLoadImageNotifyRoutine(ImageLoadNotify);
|
||||
g_ImageNotifyRegistered = FALSE;
|
||||
}
|
||||
|
||||
UninitializeRegistryProtection();
|
||||
UninitializeProcessProtection();
|
||||
|
||||
ExWaitForRundownProtectionRelease(&GlobalData.PortRundown);
|
||||
|
||||
UnloadRules();
|
||||
UninitializeRulesEngine();
|
||||
|
||||
@@ -61,12 +62,9 @@ static NTSTATUS PortMessage(PVOID PortCookie, PVOID InputBuffer, ULONG InputBuff
|
||||
if (InputBuffer && InputBufferLength >= sizeof(PYAS_USER_MESSAGE)) {
|
||||
PPYAS_USER_MESSAGE msg = (PPYAS_USER_MESSAGE)InputBuffer;
|
||||
if (msg->Command == 1 || msg->Command == 2) {
|
||||
|
||||
msg->Path[MAX_PATH_LEN - 1] = L'\0';
|
||||
|
||||
UNICODE_STRING us;
|
||||
RtlInitUnicodeString(&us, msg->Path);
|
||||
|
||||
if (us.Length > 0) {
|
||||
if (msg->Command == 1) {
|
||||
AddDynamicWhitelist(&us);
|
||||
@@ -178,22 +176,29 @@ extern "C" NTSTATUS DriverEntry(PDRIVER_OBJECT DriverObject, PUNICODE_STRING Reg
|
||||
}
|
||||
|
||||
if (!NT_SUCCESS(status)) {
|
||||
if (GlobalData.FilterHandle) {
|
||||
FltUnregisterFilter(GlobalData.FilterHandle);
|
||||
GlobalData.FilterHandle = NULL;
|
||||
}
|
||||
FltUnregisterFilter(GlobalData.FilterHandle);
|
||||
GlobalData.FilterHandle = NULL;
|
||||
UnloadRules();
|
||||
UninitializeRulesEngine();
|
||||
return status;
|
||||
}
|
||||
|
||||
InitializeProcessProtection();
|
||||
|
||||
InitializeRegistryProtection(DriverObject);
|
||||
PsSetLoadImageNotifyRoutine(ImageLoadNotify);
|
||||
|
||||
NTSTATUS imageNotifyStatus = PsSetLoadImageNotifyRoutine(ImageLoadNotify);
|
||||
if (NT_SUCCESS(imageNotifyStatus)) {
|
||||
g_ImageNotifyRegistered = TRUE;
|
||||
}
|
||||
|
||||
status = FltStartFiltering(GlobalData.FilterHandle);
|
||||
if (!NT_SUCCESS(status)) {
|
||||
PsRemoveLoadImageNotifyRoutine(ImageLoadNotify);
|
||||
if (g_ImageNotifyRegistered) {
|
||||
PsRemoveLoadImageNotifyRoutine(ImageLoadNotify);
|
||||
g_ImageNotifyRegistered = FALSE;
|
||||
}
|
||||
|
||||
UninitializeRegistryProtection();
|
||||
UninitializeProcessProtection();
|
||||
|
||||
|
||||
@@ -189,8 +189,7 @@ static OB_PREOP_CALLBACK_STATUS PreOpenThread(PVOID RegistrationContext, POB_PRE
|
||||
}
|
||||
|
||||
NTSTATUS InitializeProcessProtection() {
|
||||
static UNICODE_STRING Altitude = { 0 };
|
||||
RtlInitUnicodeString(&Altitude, L"320000");
|
||||
static UNICODE_STRING Altitude = RTL_CONSTANT_STRING(L"320000.PYAS.Ob");
|
||||
|
||||
ObCallbacks[0].ObjectType = PsProcessType;
|
||||
ObCallbacks[0].Operations = OB_OPERATION_HANDLE_CREATE | OB_OPERATION_HANDLE_DUPLICATE;
|
||||
@@ -208,7 +207,15 @@ NTSTATUS InitializeProcessProtection() {
|
||||
ObRegistration.RegistrationContext = NULL;
|
||||
ObRegistration.OperationRegistration = ObCallbacks;
|
||||
|
||||
return ObRegisterCallbacks(&ObRegistration, &ObRegistrationHandle);
|
||||
NTSTATUS status = ObRegisterCallbacks(&ObRegistration, &ObRegistrationHandle);
|
||||
|
||||
if (!NT_SUCCESS(status) && (status == STATUS_FLT_INSTANCE_ALTITUDE_COLLISION || status == STATUS_OBJECT_NAME_COLLISION)) {
|
||||
static UNICODE_STRING FallbackAltitude = RTL_CONSTANT_STRING(L"320000.PYAS.Ob.Fallback");
|
||||
ObRegistration.Altitude = FallbackAltitude;
|
||||
status = ObRegisterCallbacks(&ObRegistration, &ObRegistrationHandle);
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
VOID UninitializeProcessProtection() {
|
||||
|
||||
@@ -235,9 +235,15 @@ static NTSTATUS RegistryCallback(PVOID CallbackContext, PVOID Argument1, PVOID A
|
||||
}
|
||||
|
||||
NTSTATUS InitializeRegistryProtection(PDRIVER_OBJECT DriverObject) {
|
||||
UNICODE_STRING Altitude;
|
||||
RtlInitUnicodeString(&Altitude, L"320000");
|
||||
return CmRegisterCallbackEx(RegistryCallback, &Altitude, DriverObject, NULL, &Cookie, NULL);
|
||||
static UNICODE_STRING Altitude = RTL_CONSTANT_STRING(L"320000.PYAS.Cm");
|
||||
NTSTATUS status = CmRegisterCallbackEx(RegistryCallback, &Altitude, DriverObject, NULL, &Cookie, NULL);
|
||||
|
||||
if (!NT_SUCCESS(status)) {
|
||||
static UNICODE_STRING FallbackAltitude = RTL_CONSTANT_STRING(L"320000.PYAS.Cm.Fallback");
|
||||
status = CmRegisterCallbackEx(RegistryCallback, &FallbackAltitude, DriverObject, NULL, &Cookie, NULL);
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
VOID UninitializeRegistryProtection() {
|
||||
|
||||
@@ -525,7 +525,7 @@ BOOLEAN IsProcessTrusted(HANDLE ProcessId) {
|
||||
if (g_CacheInitialized) {
|
||||
KIRQL OldIrql;
|
||||
KeAcquireSpinLock(&TrustCacheLock, &OldIrql);
|
||||
ULONG Hash = ((ULONG)(ULONG_PTR)ProcessId) & (TRUST_CACHE_SIZE - 1);
|
||||
ULONG Hash = (((ULONG)(ULONG_PTR)ProcessId) >> 2) & (TRUST_CACHE_SIZE - 1);
|
||||
|
||||
if (TrustCache[Hash].Process == Process && TrustCache[Hash].ProcessCreateTime.QuadPart == createTime.QuadPart) {
|
||||
LARGE_INTEGER Now;
|
||||
@@ -580,7 +580,7 @@ update_cache:
|
||||
if (g_CacheInitialized) {
|
||||
KIRQL OldIrql;
|
||||
KeAcquireSpinLock(&TrustCacheLock, &OldIrql);
|
||||
ULONG Hash = ((ULONG)(ULONG_PTR)ProcessId) & (TRUST_CACHE_SIZE - 1);
|
||||
ULONG Hash = (((ULONG)(ULONG_PTR)ProcessId) >> 2) & (TRUST_CACHE_SIZE - 1);
|
||||
TrustCache[Hash].Process = Process;
|
||||
TrustCache[Hash].ProcessCreateTime = createTime;
|
||||
TrustCache[Hash].IsTrusted = isTrusted;
|
||||
|
||||
@@ -132,7 +132,7 @@ graph TD
|
||||
|
||||
The machine learning-based virus scanning engine training achieved extremely high accuracy.
|
||||
|
||||
<img width="1191" height="1323" alt="train" src="https://github.com/user-attachments/assets/2eea8426-1f82-44e3-9a38-f96f9257cf50" />
|
||||
<img width="895" height="1097" alt="train" src="https://github.com/user-attachments/assets/fa7cf0e2-01a1-4234-9358-718c2b8a2c9a" />
|
||||
|
||||
## Support System
|
||||
|
||||
|
||||
Reference in new issue
Block a user