Add migrations

Reuse existing file if content is the same

ram/consist/admin.py

@@ -21,6 +21,7 @@ class ConsistAdmin(SortableAdminBase, admin.ModelAdmin):
     list_display = ("identifier", "company", "era")
     list_filter = list_display
     search_fields = list_display
+    save_as = True
 
     fieldsets = (
         (

ram/consist/migrations/0007_alter_consist_image.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.2 on 2022-11-27 00:10
+
+from django.db import migrations, models
+import ram.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("consist", "0006_md_to_html"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="consist",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+    ]

ram/consist/models.py

@@ -4,6 +4,7 @@ from django.urls import reverse
 
 from ckeditor_uploader.fields import RichTextUploadingField
 
+from ram.utils import DeduplicatedStorage
 from metadata.models import Company, Tag
 from roster.models import RollingStock
 
@@ -17,7 +18,9 @@ class Consist(models.Model):
     )
     company = models.ForeignKey(Company, on_delete=models.CASCADE)
     era = models.CharField(max_length=32, blank=True)
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
     notes = RichTextUploadingField(blank=True)
     creation_time = models.DateTimeField(auto_now_add=True)
     updated_time = models.DateTimeField(auto_now=True)

ram/metadata/migrations/0009_alter_company_logo_alter_decoder_image_and_more.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.1.2 on 2022-11-27 00:10
+
+from django.db import migrations, models
+import ram.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("metadata", "0008_remove_decoder_interface"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="company",
+            name="logo",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="decoder",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="manufacturer",
+            name="logo",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+    ]

ram/metadata/models.py

@@ -3,7 +3,7 @@ from django.conf import settings
 from django.dispatch.dispatcher import receiver
 from django_countries.fields import CountryField
 
-from ram.utils import get_image_preview, slugify
+from ram.utils import DeduplicatedStorage, get_image_preview, slugify
 
 
 class Property(models.Model):
@@ -24,7 +24,9 @@ class Manufacturer(models.Model):
         max_length=64, choices=settings.MANUFACTURER_TYPES
     )
     website = models.URLField(blank=True)
-    logo = models.ImageField(upload_to="images/", null=True, blank=True)
+    logo = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     class Meta:
         ordering = ["category", "name"]
@@ -43,7 +45,9 @@ class Company(models.Model):
     extended_name = models.CharField(max_length=128, blank=True)
     country = CountryField()
     freelance = models.BooleanField(default=False)
-    logo = models.ImageField(upload_to="images/", null=True, blank=True)
+    logo = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     class Meta:
         verbose_name_plural = "Companies"
@@ -67,7 +71,9 @@ class Decoder(models.Model):
     )
     version = models.CharField(max_length=64, blank=True)
     sound = models.BooleanField(default=False)
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     def __str__(self):
         return "{0} - {1}".format(self.manufacturer, self.name)

ram/portal/views.py

@@ -83,7 +83,7 @@ class GetHomeFiltered(View):
             query = Q(tags__slug__iexact=search)
         else:
             raise Http404
-        rolling_stock = RollingStock.objects.filter(query).order_by(
+        rolling_stock = RollingStock.objects.filter(query).distinct().order_by(
             *order_by_fields()
         )
         matches = len(rolling_stock)

ram/ram/__init__.py

@@ -1,4 +1,4 @@
 from ram.utils import git_suffix
 
-__version__ = "0.0.20"
+__version__ = "0.0.21"
 __version__ += git_suffix(__file__)

ram/ram/utils.py

@@ -1,8 +1,29 @@
 import os
+import hashlib
 import subprocess
 
 from django.utils.html import format_html
 from django.utils.text import slugify as django_slugify
+from django.core.files.storage import FileSystemStorage
+
+
+class DeduplicatedStorage(FileSystemStorage):
+    """
+    A derived FileSystemStorage class that compares already existing files
+    (with the same name) with new uploaded ones and stores new file only if
+    sha256 hash on is content is different
+    """
+
+    def save(self, name, content, max_length=None):
+        if super().exists(name):
+            new = hashlib.sha256(content.file.getbuffer()).hexdigest()
+            with open(super().path(name), "rb") as file:
+                file_binary = file.read()
+                old = hashlib.sha256(file_binary).hexdigest()
+            if old == new:
+                return name
+
+        return super().save(name, content, max_length)
 
 
 def git_suffix(fname):

ram/roster/admin.py

@@ -125,6 +125,7 @@ class RollingStockAdmin(admin.ModelAdmin):
         "address",
         "sku",
     )
+    save_as = True
 
     fieldsets = (
         (

ram/roster/migrations/0014_alter_rollingstockdocument_file_and_more.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.1.2 on 2022-11-27 00:10
+
+from django.db import migrations, models
+import ram.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("roster", "0013_rollingstock_decoder_interface"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="rollingstockdocument",
+            name="file",
+            field=models.FileField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage(),
+                upload_to="files/",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="rollingstockimage",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+    ]

ram/roster/models.py

@@ -9,7 +9,7 @@ from django.utils.safestring import mark_safe
 
 from ckeditor_uploader.fields import RichTextUploadingField
 
-from ram.utils import get_image_preview
+from ram.utils import DeduplicatedStorage, get_image_preview
 from metadata.models import (
     Property,
     Scale,
@@ -20,11 +20,6 @@ from metadata.models import (
     RollingStockType,
 )
 
-# class OverwriteMixin(FileSystemStorage):
-#     def get_available_name(self, name, max_length):
-#         self.delete(name)
-#         return name
-
 
 class RollingClass(models.Model):
     identifier = models.CharField(max_length=128, unique=False)
@@ -137,7 +132,12 @@ class RollingStockDocument(models.Model):
         RollingStock, on_delete=models.CASCADE, related_name="document"
     )
     description = models.CharField(max_length=128, blank=True)
-    file = models.FileField(upload_to="files/", null=True, blank=True)
+    file = models.FileField(
+        upload_to="files/",
+        storage=DeduplicatedStorage(),
+        null=True,
+        blank=True,
+    )
 
     class Meta(object):
         unique_together = ("rolling_stock", "file")
@@ -158,7 +158,9 @@ class RollingStockImage(models.Model):
     rolling_stock = models.ForeignKey(
         RollingStock, on_delete=models.CASCADE, related_name="image"
     )
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
     is_thumbnail = models.BooleanField()
 
     def image_thumbnail(self):