Add migrations

Reuse existing file if content is the same

ram/consist/admin.py

@@ -21,6 +21,7 @@ class ConsistAdmin(SortableAdminBase, admin.ModelAdmin):
     list_display = ("identifier", "company", "era")
     list_filter = list_display
     search_fields = list_display
+    save_as = True
 
     fieldsets = (
         (

ram/consist/migrations/0007_alter_consist_image.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.2 on 2022-11-27 00:10
+
+from django.db import migrations, models
+import ram.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("consist", "0006_md_to_html"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="consist",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+    ]

ram/consist/models.py

@@ -4,6 +4,7 @@ from django.urls import reverse
 
 from ckeditor_uploader.fields import RichTextUploadingField
 
+from ram.utils import DeduplicatedStorage
 from metadata.models import Company, Tag
 from roster.models import RollingStock
 
@@ -17,7 +18,9 @@ class Consist(models.Model):
     )
     company = models.ForeignKey(Company, on_delete=models.CASCADE)
     era = models.CharField(max_length=32, blank=True)
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
     notes = RichTextUploadingField(blank=True)
     creation_time = models.DateTimeField(auto_now_add=True)
     updated_time = models.DateTimeField(auto_now=True)

ram/metadata/admin.py

@@ -20,8 +20,8 @@ class PropertyAdmin(admin.ModelAdmin):
 @admin.register(Decoder)
 class DecoderAdmin(admin.ModelAdmin):
     readonly_fields = ("image_thumbnail",)
-    list_display = ("__str__", "interface")
-    list_filter = ("manufacturer", "interface")
+    list_display = ("__str__", "sound")
+    list_filter = ("manufacturer", "sound")
     search_fields = ("name", "manufacturer__name")
 
 

ram/metadata/migrations/0008_remove_decoder_interface.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.2 on 2022-10-31 23:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("metadata", "0007_rename_track_scale_tracks"),
+        ("roster", "0013_rollingstock_decoder_interface"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="decoder",
+            name="interface",
+        ),
+    ]

ram/metadata/migrations/0009_alter_company_logo_alter_decoder_image_and_more.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.1.2 on 2022-11-27 00:10
+
+from django.db import migrations, models
+import ram.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("metadata", "0008_remove_decoder_interface"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="company",
+            name="logo",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="decoder",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="manufacturer",
+            name="logo",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+    ]

ram/metadata/models.py

@@ -3,7 +3,7 @@ from django.conf import settings
 from django.dispatch.dispatcher import receiver
 from django_countries.fields import CountryField
 
-from ram.utils import get_image_preview, slugify
+from ram.utils import DeduplicatedStorage, get_image_preview, slugify
 
 
 class Property(models.Model):
@@ -24,7 +24,9 @@ class Manufacturer(models.Model):
         max_length=64, choices=settings.MANUFACTURER_TYPES
     )
     website = models.URLField(blank=True)
-    logo = models.ImageField(upload_to="images/", null=True, blank=True)
+    logo = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     class Meta:
         ordering = ["category", "name"]
@@ -43,7 +45,9 @@ class Company(models.Model):
     extended_name = models.CharField(max_length=128, blank=True)
     country = CountryField()
     freelance = models.BooleanField(default=False)
-    logo = models.ImageField(upload_to="images/", null=True, blank=True)
+    logo = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     class Meta:
         verbose_name_plural = "Companies"
@@ -66,11 +70,10 @@ class Decoder(models.Model):
         limit_choices_to={"category": "model"},
     )
     version = models.CharField(max_length=64, blank=True)
-    interface = models.PositiveSmallIntegerField(
-        choices=settings.DECODER_INTERFACES, null=True, blank=True
-    )
     sound = models.BooleanField(default=False)
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
 
     def __str__(self):
         return "{0} - {1}".format(self.manufacturer, self.name)

ram/portal/templates/page.html

@@ -102,7 +102,7 @@
                       </tr>
                     </tbody>
                   </table>
-                  {% if rolling_stock.decoder %}
+                  {% if rolling_stock.decoder or rolling_stock.decoder_interface %}
                   <table class="table table-striped">
                     <thead>
                       <tr>
@@ -111,13 +111,19 @@
                     </thead>
                     <tbody>
                       <tr>
-                        <th width="35%" scope="row">Decoder</th>
+                        <th width="35%" scope="row">Interface</th>
+                        <td>{{ rolling_stock.get_decoder_interface_display }}</td>
+                      </tr>
+                      {% if rolling_stock.decoder %}
+                      <tr>
+                        <th scope="row">Decoder</th>
                         <td>{{ rolling_stock.decoder }}</td>
                       </tr>
                       <tr>
                         <th scope="row">Address</th>
                         <td>{{ rolling_stock.address }}</td>
                       </tr>
+                      {% endif %}
                     </tbody>
                   </table>
                   {% endif %}
@@ -230,6 +236,10 @@
                       </tr>
                     </thead>
                     <tbody>
+                      <tr>
+                        <th scope="row">Interface</th>
+                        <td>{{ rolling_stock.get_decoder_interface_display }}</td>
+                      </tr>
                       <tr>
                         <th width="35%" scope="row">Address</th>
                         <td>{{ rolling_stock.address }}</td>
@@ -246,10 +256,6 @@
                         <th scope="row">Version</th>
                         <td>{{ rolling_stock.decoder.version }}</td>
                       </tr>
-                      <tr>
-                        <th scope="row">Interface</th>
-                        <td>{{ rolling_stock.decoder.get_interface_display }}</td>
-                      </tr>
                       <tr>
                         <th scope="row">Sound</th>
                         <td>{{ rolling_stock.decoder.sound | yesno:"Yes,No" }}</td>

ram/portal/views.py

@@ -83,7 +83,7 @@ class GetHomeFiltered(View):
             query = Q(tags__slug__iexact=search)
         else:
             raise Http404
-        rolling_stock = RollingStock.objects.filter(query).order_by(
+        rolling_stock = RollingStock.objects.filter(query).distinct().order_by(
             *order_by_fields()
         )
         matches = len(rolling_stock)

ram/ram/__init__.py

@@ -1,4 +1,4 @@
 from ram.utils import git_suffix
 
-__version__ = "0.0.19"
+__version__ = "0.0.21"
 __version__ += git_suffix(__file__)

ram/ram/utils.py

@@ -1,8 +1,29 @@
 import os
+import hashlib
 import subprocess
 
 from django.utils.html import format_html
 from django.utils.text import slugify as django_slugify
+from django.core.files.storage import FileSystemStorage
+
+
+class DeduplicatedStorage(FileSystemStorage):
+    """
+    A derived FileSystemStorage class that compares already existing files
+    (with the same name) with new uploaded ones and stores new file only if
+    sha256 hash on is content is different
+    """
+
+    def save(self, name, content, max_length=None):
+        if super().exists(name):
+            new = hashlib.sha256(content.file.getbuffer()).hexdigest()
+            with open(super().path(name), "rb") as file:
+                file_binary = file.read()
+                old = hashlib.sha256(file_binary).hexdigest()
+            if old == new:
+                return name
+
+        return super().save(name, content, max_length)
 
 
 def git_suffix(fname):

ram/roster/admin.py

@@ -125,6 +125,7 @@ class RollingStockAdmin(admin.ModelAdmin):
         "address",
         "sku",
     )
+    save_as = True
 
     fieldsets = (
         (
@@ -148,6 +149,7 @@ class RollingStockAdmin(admin.ModelAdmin):
             "DCC",
             {
                 "fields": (
+                    "decoder_interface",
                     "decoder",
                     "address",
                 )

ram/roster/migrations/0013_rollingstock_decoder_interface.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.1 on 2022-10-31 22:27
+
+from django.db import migrations, models
+
+
+def meta_to_roster(apps, schema_editor):
+    model = apps.get_model("roster", "RollingStock")
+
+    for row in model.objects.all():
+        if row.decoder:
+            decoder_interface = row.decoder.interface
+            row.__dict__["decoder_interface"] = decoder_interface
+            row.save(update_fields=["decoder_interface"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("roster", "0012_rollingstockjournal"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="rollingstock",
+            name="decoder_interface",
+            field=models.PositiveSmallIntegerField(
+                blank=True,
+                choices=[
+                    (1, "NEM651"),
+                    (2, "NEM652"),
+                    (3, "PluX"),
+                    (4, "21MTC"),
+                    (5, "Next18/Next18S"),
+                ],
+                null=True,
+            ),
+        ),
+        migrations.RunPython(
+            meta_to_roster,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

ram/roster/migrations/0014_alter_rollingstockdocument_file_and_more.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.1.2 on 2022-11-27 00:10
+
+from django.db import migrations, models
+import ram.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("roster", "0013_rollingstock_decoder_interface"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="rollingstockdocument",
+            name="file",
+            field=models.FileField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage(),
+                upload_to="files/",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="rollingstockimage",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/",
+            ),
+        ),
+    ]

ram/roster/models.py

@@ -3,12 +3,13 @@ import re
 from uuid import uuid4
 from django.db import models
 from django.urls import reverse
+from django.conf import settings
 from django.dispatch import receiver
 from django.utils.safestring import mark_safe
 
 from ckeditor_uploader.fields import RichTextUploadingField
 
-from ram.utils import get_image_preview
+from ram.utils import DeduplicatedStorage, get_image_preview
 from metadata.models import (
     Property,
     Scale,
@@ -19,11 +20,6 @@ from metadata.models import (
     RollingStockType,
 )
 
-# class OverwriteMixin(FileSystemStorage):
-#     def get_available_name(self, name, max_length):
-#         self.delete(name)
-#         return name
-
 
 class RollingClass(models.Model):
     identifier = models.CharField(max_length=128, unique=False)
@@ -87,6 +83,9 @@ class RollingStock(models.Model):
     )
     scale = models.ForeignKey(Scale, on_delete=models.CASCADE)
     sku = models.CharField(max_length=32, blank=True)
+    decoder_interface = models.PositiveSmallIntegerField(
+        choices=settings.DECODER_INTERFACES, null=True, blank=True
+    )
     decoder = models.ForeignKey(
         Decoder, on_delete=models.CASCADE, null=True, blank=True
     )
@@ -133,7 +132,12 @@ class RollingStockDocument(models.Model):
         RollingStock, on_delete=models.CASCADE, related_name="document"
     )
     description = models.CharField(max_length=128, blank=True)
-    file = models.FileField(upload_to="files/", null=True, blank=True)
+    file = models.FileField(
+        upload_to="files/",
+        storage=DeduplicatedStorage(),
+        null=True,
+        blank=True,
+    )
 
     class Meta(object):
         unique_together = ("rolling_stock", "file")
@@ -154,7 +158,9 @@ class RollingStockImage(models.Model):
     rolling_stock = models.ForeignKey(
         RollingStock, on_delete=models.CASCADE, related_name="image"
     )
-    image = models.ImageField(upload_to="images/", null=True, blank=True)
+    image = models.ImageField(
+        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+    )
     is_thumbnail = models.BooleanField()
 
     def image_thumbnail(self):