Merge pull request #13 from daniviga/dedup

Reuse existing file if content is the same

ram/consist/models.py

@@ -4,6 +4,7 @@ from django.urls import reverse
 
 from ckeditor_uploader.fields import RichTextUploadingField
 
+from ram.utils import DeduplicatedStorage
 from metadata.models import Company, Tag
 from roster.models import RollingStock
 
@@ -17,7 +18,9 @@ class Consist(models.Model):
     )
     company = models.ForeignKey(Company, on_delete=models.CASCADE)
     era = models.CharField(max_length=32, blank=True)
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
     notes = RichTextUploadingField(blank=True)
     creation_time = models.DateTimeField(auto_now_add=True)
     updated_time = models.DateTimeField(auto_now=True)

ram/metadata/models.py

@@ -3,7 +3,7 @@ from django.conf import settings
 from django.dispatch.dispatcher import receiver
 from django_countries.fields import CountryField
 
-from ram.utils import get_image_preview, slugify
+from ram.utils import DeduplicatedStorage, get_image_preview, slugify
 
 
 class Property(models.Model):
@@ -24,7 +24,9 @@ class Manufacturer(models.Model):
         max_length=64, choices=settings.MANUFACTURER_TYPES
     )
     website = models.URLField(blank=True)
-    logo = models.ImageField(upload_to="images/", null=True, blank=True)
+    logo = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     class Meta:
         ordering = ["category", "name"]
@@ -43,7 +45,9 @@ class Company(models.Model):
     extended_name = models.CharField(max_length=128, blank=True)
     country = CountryField()
     freelance = models.BooleanField(default=False)
-    logo = models.ImageField(upload_to="images/", null=True, blank=True)
+    logo = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     class Meta:
         verbose_name_plural = "Companies"
@@ -67,7 +71,9 @@ class Decoder(models.Model):
     )
     version = models.CharField(max_length=64, blank=True)
     sound = models.BooleanField(default=False)
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     def __str__(self):
         return "{0} - {1}".format(self.manufacturer, self.name)

ram/ram/__init__.py

@@ -1,4 +1,4 @@
 from ram.utils import git_suffix
 
-__version__ = "0.0.20"
+__version__ = "0.0.21"
 __version__ += git_suffix(__file__)

ram/ram/utils.py

@@ -1,8 +1,29 @@
 import os
+import hashlib
 import subprocess
 
 from django.utils.html import format_html
 from django.utils.text import slugify as django_slugify
+from django.core.files.storage import FileSystemStorage
+
+
+class DeduplicatedStorage(FileSystemStorage):
+    """
+    A derived FileSystemStorage class that compares already existing files
+    (with the same name) with new uploaded ones and stores new file only if
+    sha256 hash on is content is different
+    """
+
+    def save(self, name, content, max_length=None):
+        if super().exists(name):
+            new = hashlib.sha256(content.file.getbuffer()).hexdigest()
+            with open(super().path(name), "rb") as file:
+                file_binary = file.read()
+                old = hashlib.sha256(file_binary).hexdigest()
+            if old == new:
+                return name
+
+        return super().save(name, content, max_length)
 
 
 def git_suffix(fname):

ram/roster/models.py

@@ -9,7 +9,7 @@ from django.utils.safestring import mark_safe
 
 from ckeditor_uploader.fields import RichTextUploadingField
 
-from ram.utils import get_image_preview
+from ram.utils import DeduplicatedStorage, get_image_preview
 from metadata.models import (
     Property,
     Scale,
@@ -20,11 +20,6 @@ from metadata.models import (
     RollingStockType,
 )
 
-# class OverwriteMixin(FileSystemStorage):
-#     def get_available_name(self, name, max_length):
-#         self.delete(name)
-#         return name
-
 
 class RollingClass(models.Model):
     identifier = models.CharField(max_length=128, unique=False)
@@ -137,7 +132,12 @@ class RollingStockDocument(models.Model):
         RollingStock, on_delete=models.CASCADE, related_name="document"
     )
     description = models.CharField(max_length=128, blank=True)
-    file = models.FileField(upload_to="files/", null=True, blank=True)
+    file = models.FileField(
+        upload_to="files/",
+        storage=DeduplicatedStorage(),
+        null=True,
+        blank=True,
+    )
 
     class Meta(object):
         unique_together = ("rolling_stock", "file")
@@ -158,7 +158,9 @@ class RollingStockImage(models.Model):
     rolling_stock = models.ForeignKey(
         RollingStock, on_delete=models.CASCADE, related_name="image"
     )
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
     is_thumbnail = models.BooleanField()
 
     def image_thumbnail(self):