Keep media folder clean (#28)

* Reorg roster, portal and bookshelf media
* Extend media reorg to consists
* Delete roster and bookshelf images on delte.
   Do not delete others data that might be dedup! 
* Bump version

ram/bookshelf/migrations/0009_alter_bookimage_image.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.2.6 on 2023-10-30 13:16
+
+import os
+import sys
+import shutil
+import ram.utils
+import bookshelf.models
+
+from django.db import migrations, models
+from django.conf import settings
+
+
+def move_images(apps, schema_editor):
+    sys.stdout.write("\n  Processing files. Please await...")
+    for r in bookshelf.models.BookImage.objects.all():
+        fname = os.path.basename(r.image.path)
+        new_image = bookshelf.models.book_image_upload(r, fname)
+        new_path = os.path.join(settings.MEDIA_ROOT, new_image)
+        os.makedirs(os.path.dirname(new_path), exist_ok=True)
+        try:
+            shutil.move(r.image.path, new_path)
+        except FileNotFoundError:
+            sys.stderr.write("  !! FileNotFoundError: {}\n".format(new_image))
+            pass
+        r.image.name = new_image
+        r.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("bookshelf", "0008_alter_author_options_alter_publisher_options"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="bookimage",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to=bookshelf.models.book_image_upload,
+            ),
+        ),
+        migrations.RunPython(
+            move_images,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

ram/bookshelf/models.py

@@ -1,3 +1,5 @@
+import os
+import shutil
 from uuid import uuid4
 from django.db import models
 from django.conf import settings
@@ -70,13 +72,31 @@ class Book(models.Model):
     def get_absolute_url(self):
         return reverse("book", kwargs={"uuid": self.uuid})
 
+    def delete(self, *args, **kwargs):
+        shutil.rmtree(
+            os.path.join(
+                settings.MEDIA_ROOT, "images", "books", str(self.uuid)
+            ),
+            ignore_errors=True
+        )
+        super(Book, self).delete(*args, **kwargs)
+
+
+def book_image_upload(instance, filename):
+    return os.path.join(
+        "images",
+        "books",
+        str(instance.book.uuid),
+        filename
+    )
+
 
 class BookImage(Image):
     book = models.ForeignKey(
         Book, on_delete=models.CASCADE, related_name="image"
     )
     image = models.ImageField(
-        upload_to="images/books/",  # FIXME, find a better way to replace this
+        upload_to=book_image_upload,
         storage=DeduplicatedStorage,
         null=True,
         blank=True

ram/consist/migrations/0009_alter_consist_image.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.2.6 on 2023-10-31 09:41
+
+import os
+import sys
+import shutil
+import ram.utils
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def move_images(apps, schema_editor):
+    sys.stdout.write("\n  Processing files. Please await...")
+    model = apps.get_model("consist", "Consist")
+    for r in model.objects.all():
+        if not r.image:  # exit the loop if there's no image
+            continue
+        fname = os.path.basename(r.image.path)
+        new_image = os.path.join("images", "consists", fname)
+        new_path = os.path.join(settings.MEDIA_ROOT, new_image)
+        os.makedirs(os.path.dirname(new_path), exist_ok=True)
+        try:
+            shutil.move(r.image.path, new_path)
+        except FileNotFoundError:
+            sys.stderr.write("  !! FileNotFoundError: {}\n".format(new_image))
+            pass
+        r.image.name = new_image
+        r.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("consist", "0008_alter_consist_options"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="consist",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/consists",
+            ),
+        ),
+        migrations.RunPython(
+            move_images,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

ram/consist/models.py

@@ -1,3 +1,5 @@
+import os
+
 from uuid import uuid4
 from django.db import models
 from django.urls import reverse
@@ -19,7 +21,10 @@ class Consist(models.Model):
     company = models.ForeignKey(Company, on_delete=models.CASCADE)
     era = models.CharField(max_length=32, blank=True)
     image = models.ImageField(
-        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+        upload_to=os.path.join("images", "consists"),
+        storage=DeduplicatedStorage,
+        null=True,
+        blank=True,
     )
     notes = RichTextUploadingField(blank=True)
     creation_time = models.DateTimeField(auto_now_add=True)

ram/metadata/migrations/0015_alter_company_logo_alter_decoder_image_and_more.py

@@ -0,0 +1,80 @@
+# Generated by Django 4.2.6 on 2023-10-30 13:16
+
+import os
+import sys
+import shutil
+import ram.utils
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def move_images(apps, schema_editor):
+    fields = {
+        "Company": ["companies", "logo"],
+        "Decoder": ["decoders", "image"],
+        "Manufacturer": ["manufacturers", "logo"],
+    }
+    sys.stdout.write("\n  Processing files. Please await...")
+    for m in fields.items():
+        model = apps.get_model("metadata", m[0])
+        for r in model.objects.all():
+            field = getattr(r, m[1][1])
+            if not field:  # exit the loop if there's no image
+                continue
+            fname = os.path.basename(field.path)
+            new_image = os.path.join("images", m[1][0], fname)
+            new_path = os.path.join(settings.MEDIA_ROOT, new_image)
+            os.makedirs(os.path.dirname(new_path), exist_ok=True)
+            try:
+                shutil.move(field.path, new_path)
+            except FileNotFoundError:
+                sys.stderr.write(
+                    "  !! FileNotFoundError: {}\n".format(new_image)
+                )
+                pass
+            field.name = new_image
+            r.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("metadata", "0014_alter_decoder_options_alter_tag_options"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="company",
+            name="logo",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/companies",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="decoder",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/decoders",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="manufacturer",
+            name="logo",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to="images/manufacturers",
+            ),
+        ),
+        migrations.RunPython(
+            move_images,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

ram/metadata/models.py

@@ -1,3 +1,4 @@
+import os
 from django.db import models
 from django.urls import reverse
 from django.conf import settings
@@ -28,7 +29,10 @@ class Manufacturer(models.Model):
     )
     website = models.URLField(blank=True)
     logo = models.ImageField(
-        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+        upload_to=os.path.join("images", "manufacturers"),
+        storage=DeduplicatedStorage,
+        null=True,
+        blank=True,
     )
 
     class Meta:
@@ -58,7 +62,10 @@ class Company(models.Model):
     country = CountryField()
     freelance = models.BooleanField(default=False)
     logo = models.ImageField(
-        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+        upload_to=os.path.join("images", "companies"),
+        storage=DeduplicatedStorage,
+        null=True,
+        blank=True,
     )
 
     class Meta:
@@ -95,7 +102,10 @@ class Decoder(models.Model):
     version = models.CharField(max_length=64, blank=True)
     sound = models.BooleanField(default=False)
     image = models.ImageField(
-        upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True
+        upload_to=os.path.join("images", "decoders"),
+        storage=DeduplicatedStorage,
+        null=True,
+        blank=True,
     )
 
     class Meta(object):

ram/portal/templates/cards/book.html

@@ -1,7 +1,7 @@
         <div class="col">
           <div class="card shadow-sm">
             {% if d.item.image.exists %}
-            <a href="{{d.item.get_absolute_url}}"><img class="card-img-top" src="{{ d.item.image.first.image.url }}" alt="{{ d }}"></a>
+            <a href="{{d.item.get_absolute_url}}"><img class="card-img-top" src="{{ d.item.image.first.image.url }}" alt="{{ d.item }}"></a>
             {% endif %}
             <div class="card-body">
               <p class="card-text" style="position: relative;">

ram/portal/templates/cards/consist.html

@@ -2,10 +2,10 @@
         <div class="card shadow-sm">
           <a href="{{ d.item.get_absolute_url }}">
           {% if d.item.image %}
-            <img class="card-img-top" src="{{ d.item.image.url }}" alt="{{ d }}">
+            <img class="card-img-top" src="{{ d.item.image.url }}" alt="{{ d.item }}">
           {% else %}
             {% with d.item.consist_item.first.rolling_stock as r %}
-            <img class="card-img-top" src="{{ r.image.first.image.url }}" alt="{{ d }}">
+            <img class="card-img-top" src="{{ r.image.first.image.url }}" alt="{{ d.item }}">
             {% endwith %}
           {% endif %}
           </a>

ram/portal/templates/cards/roster.html

@@ -2,10 +2,10 @@
         <div class="col">
           <div class="card shadow-sm">
             {% if d.item.image.exists %}
-            <a href="{{d.item.get_absolute_url}}"><img class="card-img-top" src="{{ d.item.image.first.image.url }}" alt="{{ d }}"></a>
+            <a href="{{d.item.get_absolute_url}}"><img class="card-img-top" src="{{ d.item.image.first.image.url }}" alt="{{ d.item }}"></a>
             {% else %}
             <!-- Do not show the "Coming soon" image when running in a single card column mode (e.g. on mobile) -->
-            <a href="{{d.item.get_absolute_url}}"><img class="card-img-top d-none d-sm-block" src="{% static DEFAULT_CARD_IMAGE %}" alt="{{ d }}"></a>
+            <a href="{{d.item.get_absolute_url}}"><img class="card-img-top d-none d-sm-block" src="{% static DEFAULT_CARD_IMAGE %}" alt="{{ d.item }}"></a>
             {% endif %}
             <div class="card-body">
               <p class="card-text" style="position: relative;">

ram/ram/__init__.py

@@ -1,4 +1,4 @@
 from ram.utils import git_suffix
 
-__version__ = "0.9.3"
+__version__ = "0.9.4"
 __version__ += git_suffix(__file__)

ram/roster/migrations/0020_alter_rollingstockimage_image.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.2.6 on 2023-10-30 13:16
+
+import os
+import sys
+import shutil
+import ram.utils
+import roster.models
+
+from django.db import migrations, models
+from django.conf import settings
+
+
+def move_images(apps, schema_editor):
+    sys.stdout.write("\n  Processing files. Please await...")
+    for r in roster.models.RollingStockImage.objects.all():
+        fname = os.path.basename(r.image.path)
+        new_image = roster.models.rolling_stock_image_upload(r, fname)
+        new_path = os.path.join(settings.MEDIA_ROOT, new_image)
+        os.makedirs(os.path.dirname(new_path), exist_ok=True)
+        try:
+            shutil.move(r.image.path, new_path)
+        except FileNotFoundError:
+            sys.stderr.write("  !! FileNotFoundError: {}\n".format(new_image))
+            pass
+        r.image.name = new_image
+        r.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("roster", "0019_rollingstockdocument_private"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="rollingstockimage",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                storage=ram.utils.DeduplicatedStorage,
+                upload_to=roster.models.rolling_stock_image_upload,
+            ),
+        ),
+        migrations.RunPython(
+            move_images,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

ram/roster/models.py

@@ -1,4 +1,6 @@
+import os
 import re
+import shutil
 from uuid import uuid4
 from django.db import models
 from django.urls import reverse
@@ -8,7 +10,7 @@ from django.dispatch import receiver
 from ckeditor_uploader.fields import RichTextUploadingField
 
 from ram.models import Document, Image, PropertyInstance
-from ram.utils import get_image_preview
+from ram.utils import DeduplicatedStorage
 from metadata.models import (
     Scale,
     Manufacturer,
@@ -106,6 +108,15 @@ class RollingStock(models.Model):
     def company(self):
         return str(self.rolling_class.company)
 
+    def delete(self, *args, **kwargs):
+        shutil.rmtree(
+            os.path.join(
+                settings.MEDIA_ROOT, "images", "rollingstock", str(self.uuid)
+            ),
+            ignore_errors=True
+        )
+        super(RollingStock, self).delete(*args, **kwargs)
+
 
 @receiver(models.signals.pre_save, sender=RollingStock)
 def pre_save_running_number(sender, instance, *args, **kwargs):
@@ -126,10 +137,25 @@ class RollingStockDocument(Document):
         unique_together = ("rolling_stock", "file")
 
 
+def rolling_stock_image_upload(instance, filename):
+    return os.path.join(
+        "images",
+        "rollingstock",
+        str(instance.rolling_stock.uuid),
+        filename
+    )
+
+
 class RollingStockImage(Image):
     rolling_stock = models.ForeignKey(
         RollingStock, on_delete=models.CASCADE, related_name="image"
     )
+    image = models.ImageField(
+        upload_to=rolling_stock_image_upload,
+        storage=DeduplicatedStorage,
+        null=True,
+        blank=True,
+    )
 
 
 class RollingStockProperty(PropertyInstance):