Add files via upload

PYAS.py

@@ -906,6 +906,7 @@ class MainWindow_Controller(QMainWindow):
                             pass
             QApplication.processEvents()
             ones, zero = self.pe.predict(fn)
+            #print(ones, zero)
             if self.json["high_sensitive"]:
                 return ones >= zero
             elif "_CorExeMain" not in fn:

PYAS_Engine.py

@@ -1,9 +1,9 @@
-from simhash import Simhash
-import json
+import json, re, hashlib, numpy
+from itertools import groupby
 
 class ListSimHash:
     def __init__(self):
-        self.model = {"ones": [], "zero": []}
+        self.model = {}
 
     def save_model(self, file_name):
         with open(file_name, 'w') as f:
@@ -16,38 +16,41 @@ class ListSimHash:
                 model_data = json.load(f)
         self.model = model_data
 
-    def train_model(self, ones, zero):
-        self.model["ones"] = [Simhash(" ".join(x)).value for x in ones]
-        self.model["zero"] = [Simhash(" ".join(y)).value for y in zero]
+    def train_model(self, ones, zero, feature=128, batch_size=200):
+        self.model["feature"] = feature
+        self.model["batch_size"] = batch_size
+        self.model["ones"] = [self.build_text(" ".join(x)) for x in ones]
+        self.model["zero"] = [self.build_text(" ".join(y)) for y in zero]
 
-    def predict(self, query, length=64):
-        query_hash = Simhash(" ".join(query)).value
-        max_ones = max(self.similar(x, query_hash, length) for x in self.model["ones"])
-        max_zero = max(self.similar(y, query_hash, length) for y in self.model["zero"])
+    def build_text(self, content, width=4):
+        sums, batch, count = [], [], 0
+        content = ''.join(re.findall(r'[\w\u4e00-\u9fcc]+', content.lower()))
+        features = [content[i:i + width] for i in range(max(len(content) - width + 1, 1))]
+        features = {k: sum(1 for _ in g) for k, g in groupby(sorted(features))}
+        for f, w in features.items():
+            count += w
+            h = hashlib.md5(f.encode('utf-8')).digest()
+            batch.append(h * w)
+            if len(batch) >= self.model["batch_size"]:
+                sums.append(self.sum_hashes(batch))
+                batch = []
+        if batch:
+            sums.append(self.sum_hashes(batch))
+        combined_sums = numpy.sum(sums, 0)
+        v = numpy.packbits(combined_sums > count / 2).tobytes()
+        return int.from_bytes(v, 'big')
+
+    def sum_hashes(self, digests):
+        bitarray = numpy.unpackbits(numpy.frombuffer(b''.join(digests), dtype='>B'))
+        return numpy.sum(numpy.reshape(bitarray, (-1, self.model["feature"])), 0)
+
+    def predict(self, query):
+        query_hash = self.build_text(" ".join(query))
+        max_ones = max(self.similar(x, query_hash) for x in self.model["ones"])
+        max_zero = max(self.similar(y, query_hash) for y in self.model["zero"])
         return max_ones, max_zero
 
-    def similar(self, x, y, length):
+    def similar(self, x, y):
         hamming_distance = bin(x ^ y).count('1')
-        similarity = 1 - hamming_distance / length
+        similarity = 1 - hamming_distance / self.model["feature"]
         return similarity
-
-############################################################
-
-'''
-
-ones = [['a', 'b', 'c'], ['d', 'e', 'f']]
-zero = [['1', '2', '3'], ['4', '5', '6']]
-
-simhash_obj = ListSimHash()
-simhash_obj.train_model(ones, zero)
-simhash_obj.save_model('Model.json')
-
-news = ['a', 'b', 'c', 'd']
-
-simhash_obj = ListSimHash()
-simhash_obj.load_model('Model.json')
-print(simhash_obj.predict(news))
-
-'''
-
-############################################################

PYAS_Model.json

@@ -1 +1 @@
-{"ones": [5506179105492874788, 7517035770115539694, 13882879675322517133, 7818799172664193602, 1973113986374748014, 14445724089571362409, 2950398409228757597, 3107503436321588775, 15877480836211565093, 9903908802304460398, 15877480836219953701, 14805764333567894053, 14481813049420702373, 10155727276557623461, 6270388326867688046, 4949860217863096401, 6688304182173618410, 15956530817393910309, 885491361875663726, 11384735600477659757, 14731683634372567597, 3227150962161735406, 5497404648499036334, 8749349020663826286, 6734329188654170793, 6569173175630607439, 14445555523920519717, 13941653725777256617, 14733847473260225197, 5986938328694501706, 15671106351935185453, 14444367982643012141, 5508366968215268012, 3294629996004656684, 13923340762616325677, 9851875522372698253, 13900919586430534308, 11721377299457837, 13902045417079456428, 13869094772153935404, 16535164161726243384, 14477296689947624613, 14445460278959039072, 14480326514542272549, 15599524867940475901, 13923340762616325677, 5751854475000252650, 13975220046243178029, 9188382994275353314, 14769542018744345133, 7818447241834435210, 9867664636587163817, 14192364127870495415, 5504776031945810605, 14689137838120330033, 13863483345808540332, 15670684156647834156, 14066325786815516204, 13867960076154044973, 14447024935311647277, 14443585753671156269, 14443181129105568293, 14444297721818276005, 15659900129458414137, 5251461282652410188, 4645938775912096265, 14435625265326866465, 13870229468153833005, 14734568735708177005, 9227187293395161634, 4460993081932304610, 14443857878504104493, 5233607121500054758, 13923340762616325677, 15669611569668817453, 14444438411516732077, 14553857952375530173, 13862357549014475309, 15668432356867719725, 14733724310778044973, 14444297687458538021, 15875580334657921585, 4928526265114616749, 605002848261333645, 13863764820246282924, 16747045017347350624, 14554971761381089964, 5318651344351293544, 13867142593016947245, 14443445016182801197, 7589678793269660174, 15877489632310878757, 14519029821519716973, 14445564251294016173, 14438817721505686061, 14443172333012541989, 14589851821875281957, 14590596260537197735, 278005917486916355, 15591739226110435885, 14443220698093978407, 8043324694489192651, 13902045417079456428, 13870212425723553325, 9813173604152735417, 14731552268507046437, 29595016310973060, 5216686530752835469, 3024998408763818636, 14443163541206090541, 5217249222473466925, 14501982374758604841, 3016713856541827211, 5222395147315478669, 5216400674449254029, 14443383218045863069, 16224957599475629, 14434218996397393449, 13863764820783153836, 14479622818508464293, 13974103011653730988, 14803609312787063333, 17939443371365261477, 14434587349972574241, 7628446096252719673, 14722448260689437349, 14443312511609889453, 14444438420106666541, 5612861735302520877, 14734067409383989925, 14515510856555852332, 14443436220089778989, 4640223596068202157, 14515378915137446437, 5508373564771181101, 14443603070987674157, 14471389627660247725, 5223371286573934221, 14515863228559811325, 13913505996159493672, 5542845041004143277, 5216618361034106381, 4162844041526332072, 4694411910500052621, 15616694275657834661, 5504919312161825709, 5794061389005218345, 14446690220457222701, 14517613667710034445, 5221119624199202445, 14445828207099666733, 5216686530752832269, 14734551143522165165, 14479201164382855213, 9867664636587163820, 599812131744310892, 592756094753058605, 10014058044927534692, 5216686530752835213, 5216684331201159853, 5395787407074543118, 14733565997746651689, 2932112211939017308, 7033423939598762665, 16905834591490556004, 14443180510057738789, 15884750278813446701, 14731683569952239653, 5330542626256339789, 10984548548082024492, 14443445016182800941, 14057185508926367584, 5217531859754894316, 5216682132775592845, 14444070071363330613, 14481364422908863017, 7636176397580752551, 13938074214637388460, 13862638989095092781, 14443602791814800037, 9942296777753215662, 5581338874910041517, 14445573120401498157, 304103761821198221, 4934155884906309293, 14445766665049629349, 14580950141956391013, 5216681582415733645, 2019280485283460812, 13870229330714879533, 14444280094693671521, 14445406540094907300, 14518110651273208493, 5216682132169453453, 9903943982901675630, 16165007526532104225, 5216682132773495693, 14723038741383217701, 7578679156640474205, 4928596617278777005, 9327450245803134830, 13974103011653730988, 14733926642392392225, 5216686530752835213, 13869112913559437989, 13857853914488914477, 14443321801624150573, 13866861113745228333, 14734270217764364980, 14434218996397393441, 14445489490543117605, 14733794563556043305, 5217753194477011109, 5219078397515718701, 5510352621782066729, 5216686530752834445, 13863764820246282924, 14803371783915708961, 5216686530752832396, 4928596754659010221, 15633327149618192937, 13847901066272048685, 892670137082570509, 14428315735651917984, 14479618330261348516, 6384061068061721550, 9865615069570031940, 5216688454369796749, 14443242693164671525, 13869394389039465125, 13864538949482332717, 13963257577836Line truncated
+{"feature": 128, "batch_size": 200, "ones": [138884061479927293269157399561321625124, 138514579140422510314615591404235408110, 56400517596638244422894049797604820621, 216322613833371885512412681366080352834, 57509758994800025148320631233619478382, 106913533745562494950237068000717194857, 268764573572222081482569414853338147421, 301824928671740406049708617676147983911, 139390375862009821188813492031092843045, 85423552611944120389541188728576853614, 96855081274446633945978062387702817317, 266907975564823269757642468520682218021, 145963460354456926129955518358155059877, 310175323171803986711996189701871919269, 130213060982497483351391950606798445166, 804810740308425468096265735208257617, 212959202992585676402479957744392065258, 139400638929619405534171854486176686629, 322017788596429201866715226549004966766, 140636369840301821837859972310736757357, 309443250354319958579894353439906751021, 93525514883026399429643965685843744494, 305869899604823982125728318036880769198, 100726355023313678737270049805708838766, 128087963732730699244600678440497496745, 42649730517016866640152470265639253071, 310789353473439559353693532556983984677, 310834785753961546009345771577019752617, 98101030488836797416915605797123547821, 163055950804159105041684713458861453642, 140783818610473943253735333316647117357, 267073946987621142567955636205481517613, 308840950453806983092423514217084578476, 131790595577365633473641510877883811372, 314740690747489056417995540920077280813, 312173101091522310981766359248219144333, 266919150567781465017126522717174383268, 98595718339270037315648519699911042861, 266908765191386477077819778373267636908, 98097114716132427630865195752911955500, 309174548517306493765742812127600992824, 165719174729833763978390369816514151589, 333214357055760258309229460312077263456, 309443006965336732125112839751851204645, 138794485055151379651410532066913608701, 314740690747489056417995540920077280813, 163777668652403995261587270005386479850, 314760122553781807187305329030403597869, 236708587517087268685876607084596151010, 309453635424025901100964431663839207981, 130453416906105788293089319006359032458, 144623683762647410432822459404215795881, 122948904762874323200167759687292791479, 141960523677151839211304480019230155437, 175281760716791954210630508746562429745, 266908927450643968163503768535437820588, 309526327094009133336072658832466510380, 147109497140790587393132625769962294828, 309443088174281869980853203902500569645, 98179117726185629973687845690377904685, 266907874033546142647711315245141475885, 139317724478784922140315175893335437861, 309460205972074921952248716662523977893, 266898908365541525694120496365498163769, 128833381668692705604881271753964704076, 308755229979748206303057066098374746633, 312110631309585432258276350165909913633, 96850943187510051473774485962306807341, 266991032348753532132223164567115948653, 301704982992736118799149217975167423010, 45423085047571636176304357087088791778, 98185566304403225600012489980760220205, 301974605256105130746947982559953757414, 314740690747489056417995540920077280813, 288187285762824132116353144464193487405, 309443331008521396576751777765001683629, 272224825421632604778319829685274829501, 266919475084642627997330537792153215533, 309527625168214095599527134475951948333, 268325411442133354907118265595969561133, 309459962424693681697118037040663257637, 309517280516176477337549624053815730737, 123342205008290812825969949820881134509, 138980104556989140895595181268125835917, 266908765349823459293196338896727065260, 118095062731544531316068994170817536096, 140693602135645422745421550777944908460, 440767454710337817086060213441032296, 266887023150038950813539942248450810413, 267006527672619154632785007938033304365, 46088979409888770325991887802833783310, 96856378081010667424655802311172842021, 98075229203956313888405997203836658285, 96783566280022178361451345888136274605, 309532816881431070079891773455787762221, 139307136862901466424877793378792006181, 96753309757095605084115148461573554213, 288162135445850533606754792760600323239, 334364013698977648789398548127408169731, 309444547834874662752143629118327890477, 298809467903569967359434970965896200999, 136838710717061085016646030313491257547, 288176899901775619274666676430489666220, 96849887669860409113990554458904733229, 132687201639749430771754766010675328697, 96865343064503306525844664637493502501, 140889247564697972421510238399255437956, 120368316428277902366824346477250637709, 138478276465840759527093181786120602252, 267006527672619154632784726463056593709, 120204759374330029658546372684809684013, 309541905261615072912890344878225383465, 266750361825108195839408748923433296011, 141949813792938456152307200983465099405, 122831899028016154140298160370331674253, 266243137498865212396848583134231807133, 119695914678333910726669088361572362157, 309537848216162082687265642225828513321, 266908765349842802069416684816640131756, 266923287357511288958920881421883433125, 266909251929602884051145652263806452396, 1394006389296194055Line truncated