Add files via upload

Engine/Model/deduplicate.py

@@ -0,0 +1,29 @@
+import os, hashlib
+
+def calc_sha256(path):
+    with open(path, "rb") as f:
+        return hashlib.sha256(f.read()).digest()
+
+def dedup_realtime(root):
+    seen = set()
+    deleted = 0
+    for dirpath, _, files in os.walk(root):
+        for name in files:
+            p = os.path.join(dirpath, name)
+            try:
+                h = calc_sha256(p)
+            except Exception:
+                continue
+            if h in seen:
+                try:
+                    os.remove(p)
+                    print(p)
+                    deleted += 1
+                except Exception:
+                    pass
+            else:
+                seen.add(h)
+    print(f"deleted {deleted}")
+
+if __name__ == "__main__":
+    dedup_realtime(r".\Image_File")