From 830da80302492e328c3cc44adfba339d2aa61ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniele=20Vigan=C3=B2?= Date: Tue, 31 Oct 2023 11:16:55 +0100 Subject: [PATCH] Keep media folder clean (#28) * Reorg roster, portal and bookshelf media * Extend media reorg to consists * Delete roster and bookshelf images on delte. Do not delete others data that might be dedup! * Bump version --- .../migrations/0009_alter_bookimage_image.py | 49 ++++++++++++ ram/bookshelf/models.py | 22 ++++- .../migrations/0009_alter_consist_image.py | 51 ++++++++++++ ram/consist/models.py | 7 +- ...mpany_logo_alter_decoder_image_and_more.py | 80 +++++++++++++++++++ ram/metadata/models.py | 16 +++- ram/portal/templates/cards/book.html | 2 +- ram/portal/templates/cards/consist.html | 4 +- ram/portal/templates/cards/roster.html | 4 +- ram/ram/__init__.py | 2 +- .../0020_alter_rollingstockimage_image.py | 49 ++++++++++++ ram/roster/models.py | 28 ++++++- 12 files changed, 302 insertions(+), 12 deletions(-) create mode 100644 ram/bookshelf/migrations/0009_alter_bookimage_image.py create mode 100644 ram/consist/migrations/0009_alter_consist_image.py create mode 100644 ram/metadata/migrations/0015_alter_company_logo_alter_decoder_image_and_more.py create mode 100644 ram/roster/migrations/0020_alter_rollingstockimage_image.py diff --git a/ram/bookshelf/migrations/0009_alter_bookimage_image.py b/ram/bookshelf/migrations/0009_alter_bookimage_image.py new file mode 100644 index 0000000..0a50733 --- /dev/null +++ b/ram/bookshelf/migrations/0009_alter_bookimage_image.py @@ -0,0 +1,49 @@ +# Generated by Django 4.2.6 on 2023-10-30 13:16 + +import os +import sys +import shutil +import ram.utils +import bookshelf.models + +from django.db import migrations, models +from django.conf import settings + + +def move_images(apps, schema_editor): + sys.stdout.write("\n Processing files. Please await...") + for r in bookshelf.models.BookImage.objects.all(): + fname = os.path.basename(r.image.path) + new_image = bookshelf.models.book_image_upload(r, fname) + new_path = os.path.join(settings.MEDIA_ROOT, new_image) + os.makedirs(os.path.dirname(new_path), exist_ok=True) + try: + shutil.move(r.image.path, new_path) + except FileNotFoundError: + sys.stderr.write(" !! FileNotFoundError: {}\n".format(new_image)) + pass + r.image.name = new_image + r.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("bookshelf", "0008_alter_author_options_alter_publisher_options"), + ] + + operations = [ + migrations.AlterField( + model_name="bookimage", + name="image", + field=models.ImageField( + blank=True, + null=True, + storage=ram.utils.DeduplicatedStorage, + upload_to=bookshelf.models.book_image_upload, + ), + ), + migrations.RunPython( + move_images, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/ram/bookshelf/models.py b/ram/bookshelf/models.py index 0d01d10..4785c3b 100644 --- a/ram/bookshelf/models.py +++ b/ram/bookshelf/models.py @@ -1,3 +1,5 @@ +import os +import shutil from uuid import uuid4 from django.db import models from django.conf import settings @@ -70,13 +72,31 @@ class Book(models.Model): def get_absolute_url(self): return reverse("book", kwargs={"uuid": self.uuid}) + def delete(self, *args, **kwargs): + shutil.rmtree( + os.path.join( + settings.MEDIA_ROOT, "images", "books", str(self.uuid) + ), + ignore_errors=True + ) + super(Book, self).delete(*args, **kwargs) + + +def book_image_upload(instance, filename): + return os.path.join( + "images", + "books", + str(instance.book.uuid), + filename + ) + class BookImage(Image): book = models.ForeignKey( Book, on_delete=models.CASCADE, related_name="image" ) image = models.ImageField( - upload_to="images/books/", # FIXME, find a better way to replace this + upload_to=book_image_upload, storage=DeduplicatedStorage, null=True, blank=True diff --git a/ram/consist/migrations/0009_alter_consist_image.py b/ram/consist/migrations/0009_alter_consist_image.py new file mode 100644 index 0000000..1a30d13 --- /dev/null +++ b/ram/consist/migrations/0009_alter_consist_image.py @@ -0,0 +1,51 @@ +# Generated by Django 4.2.6 on 2023-10-31 09:41 + +import os +import sys +import shutil +import ram.utils + +from django.conf import settings +from django.db import migrations, models + + +def move_images(apps, schema_editor): + sys.stdout.write("\n Processing files. Please await...") + model = apps.get_model("consist", "Consist") + for r in model.objects.all(): + if not r.image: # exit the loop if there's no image + continue + fname = os.path.basename(r.image.path) + new_image = os.path.join("images", "consists", fname) + new_path = os.path.join(settings.MEDIA_ROOT, new_image) + os.makedirs(os.path.dirname(new_path), exist_ok=True) + try: + shutil.move(r.image.path, new_path) + except FileNotFoundError: + sys.stderr.write(" !! FileNotFoundError: {}\n".format(new_image)) + pass + r.image.name = new_image + r.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("consist", "0008_alter_consist_options"), + ] + + operations = [ + migrations.AlterField( + model_name="consist", + name="image", + field=models.ImageField( + blank=True, + null=True, + storage=ram.utils.DeduplicatedStorage, + upload_to="images/consists", + ), + ), + migrations.RunPython( + move_images, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/ram/consist/models.py b/ram/consist/models.py index a21f78d..f0e27c0 100644 --- a/ram/consist/models.py +++ b/ram/consist/models.py @@ -1,3 +1,5 @@ +import os + from uuid import uuid4 from django.db import models from django.urls import reverse @@ -19,7 +21,10 @@ class Consist(models.Model): company = models.ForeignKey(Company, on_delete=models.CASCADE) era = models.CharField(max_length=32, blank=True) image = models.ImageField( - upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True + upload_to=os.path.join("images", "consists"), + storage=DeduplicatedStorage, + null=True, + blank=True, ) notes = RichTextUploadingField(blank=True) creation_time = models.DateTimeField(auto_now_add=True) diff --git a/ram/metadata/migrations/0015_alter_company_logo_alter_decoder_image_and_more.py b/ram/metadata/migrations/0015_alter_company_logo_alter_decoder_image_and_more.py new file mode 100644 index 0000000..ee0d04a --- /dev/null +++ b/ram/metadata/migrations/0015_alter_company_logo_alter_decoder_image_and_more.py @@ -0,0 +1,80 @@ +# Generated by Django 4.2.6 on 2023-10-30 13:16 + +import os +import sys +import shutil +import ram.utils + +from django.conf import settings +from django.db import migrations, models + + +def move_images(apps, schema_editor): + fields = { + "Company": ["companies", "logo"], + "Decoder": ["decoders", "image"], + "Manufacturer": ["manufacturers", "logo"], + } + sys.stdout.write("\n Processing files. Please await...") + for m in fields.items(): + model = apps.get_model("metadata", m[0]) + for r in model.objects.all(): + field = getattr(r, m[1][1]) + if not field: # exit the loop if there's no image + continue + fname = os.path.basename(field.path) + new_image = os.path.join("images", m[1][0], fname) + new_path = os.path.join(settings.MEDIA_ROOT, new_image) + os.makedirs(os.path.dirname(new_path), exist_ok=True) + try: + shutil.move(field.path, new_path) + except FileNotFoundError: + sys.stderr.write( + " !! FileNotFoundError: {}\n".format(new_image) + ) + pass + field.name = new_image + r.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("metadata", "0014_alter_decoder_options_alter_tag_options"), + ] + + operations = [ + migrations.AlterField( + model_name="company", + name="logo", + field=models.ImageField( + blank=True, + null=True, + storage=ram.utils.DeduplicatedStorage, + upload_to="images/companies", + ), + ), + migrations.AlterField( + model_name="decoder", + name="image", + field=models.ImageField( + blank=True, + null=True, + storage=ram.utils.DeduplicatedStorage, + upload_to="images/decoders", + ), + ), + migrations.AlterField( + model_name="manufacturer", + name="logo", + field=models.ImageField( + blank=True, + null=True, + storage=ram.utils.DeduplicatedStorage, + upload_to="images/manufacturers", + ), + ), + migrations.RunPython( + move_images, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/ram/metadata/models.py b/ram/metadata/models.py index f6acae1..a6032e4 100644 --- a/ram/metadata/models.py +++ b/ram/metadata/models.py @@ -1,3 +1,4 @@ +import os from django.db import models from django.urls import reverse from django.conf import settings @@ -28,7 +29,10 @@ class Manufacturer(models.Model): ) website = models.URLField(blank=True) logo = models.ImageField( - upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True + upload_to=os.path.join("images", "manufacturers"), + storage=DeduplicatedStorage, + null=True, + blank=True, ) class Meta: @@ -58,7 +62,10 @@ class Company(models.Model): country = CountryField() freelance = models.BooleanField(default=False) logo = models.ImageField( - upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True + upload_to=os.path.join("images", "companies"), + storage=DeduplicatedStorage, + null=True, + blank=True, ) class Meta: @@ -95,7 +102,10 @@ class Decoder(models.Model): version = models.CharField(max_length=64, blank=True) sound = models.BooleanField(default=False) image = models.ImageField( - upload_to="images/", storage=DeduplicatedStorage, null=True, blank=True + upload_to=os.path.join("images", "decoders"), + storage=DeduplicatedStorage, + null=True, + blank=True, ) class Meta(object): diff --git a/ram/portal/templates/cards/book.html b/ram/portal/templates/cards/book.html index d34fd0c..a74c7d4 100644 --- a/ram/portal/templates/cards/book.html +++ b/ram/portal/templates/cards/book.html @@ -1,7 +1,7 @@
{% if d.item.image.exists %} - {{ d }} + {{ d.item }} {% endif %}

diff --git a/ram/portal/templates/cards/consist.html b/ram/portal/templates/cards/consist.html index 4c9fb8b..4caffa2 100644 --- a/ram/portal/templates/cards/consist.html +++ b/ram/portal/templates/cards/consist.html @@ -2,10 +2,10 @@

{% if d.item.image %} - {{ d }} + {{ d.item }} {% else %} {% with d.item.consist_item.first.rolling_stock as r %} - {{ d }} + {{ d.item }} {% endwith %} {% endif %} diff --git a/ram/portal/templates/cards/roster.html b/ram/portal/templates/cards/roster.html index f82db4c..09ba0d0 100644 --- a/ram/portal/templates/cards/roster.html +++ b/ram/portal/templates/cards/roster.html @@ -2,10 +2,10 @@
{% if d.item.image.exists %} - {{ d }} + {{ d.item }} {% else %} - {{ d }} + {{ d.item }} {% endif %}

diff --git a/ram/ram/__init__.py b/ram/ram/__init__.py index 5a8c832..51211d6 100644 --- a/ram/ram/__init__.py +++ b/ram/ram/__init__.py @@ -1,4 +1,4 @@ from ram.utils import git_suffix -__version__ = "0.9.3" +__version__ = "0.9.4" __version__ += git_suffix(__file__) diff --git a/ram/roster/migrations/0020_alter_rollingstockimage_image.py b/ram/roster/migrations/0020_alter_rollingstockimage_image.py new file mode 100644 index 0000000..c6b5181 --- /dev/null +++ b/ram/roster/migrations/0020_alter_rollingstockimage_image.py @@ -0,0 +1,49 @@ +# Generated by Django 4.2.6 on 2023-10-30 13:16 + +import os +import sys +import shutil +import ram.utils +import roster.models + +from django.db import migrations, models +from django.conf import settings + + +def move_images(apps, schema_editor): + sys.stdout.write("\n Processing files. Please await...") + for r in roster.models.RollingStockImage.objects.all(): + fname = os.path.basename(r.image.path) + new_image = roster.models.rolling_stock_image_upload(r, fname) + new_path = os.path.join(settings.MEDIA_ROOT, new_image) + os.makedirs(os.path.dirname(new_path), exist_ok=True) + try: + shutil.move(r.image.path, new_path) + except FileNotFoundError: + sys.stderr.write(" !! FileNotFoundError: {}\n".format(new_image)) + pass + r.image.name = new_image + r.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("roster", "0019_rollingstockdocument_private"), + ] + + operations = [ + migrations.AlterField( + model_name="rollingstockimage", + name="image", + field=models.ImageField( + blank=True, + null=True, + storage=ram.utils.DeduplicatedStorage, + upload_to=roster.models.rolling_stock_image_upload, + ), + ), + migrations.RunPython( + move_images, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/ram/roster/models.py b/ram/roster/models.py index 1573c4c..3268335 100644 --- a/ram/roster/models.py +++ b/ram/roster/models.py @@ -1,4 +1,6 @@ +import os import re +import shutil from uuid import uuid4 from django.db import models from django.urls import reverse @@ -8,7 +10,7 @@ from django.dispatch import receiver from ckeditor_uploader.fields import RichTextUploadingField from ram.models import Document, Image, PropertyInstance -from ram.utils import get_image_preview +from ram.utils import DeduplicatedStorage from metadata.models import ( Scale, Manufacturer, @@ -106,6 +108,15 @@ class RollingStock(models.Model): def company(self): return str(self.rolling_class.company) + def delete(self, *args, **kwargs): + shutil.rmtree( + os.path.join( + settings.MEDIA_ROOT, "images", "rollingstock", str(self.uuid) + ), + ignore_errors=True + ) + super(RollingStock, self).delete(*args, **kwargs) + @receiver(models.signals.pre_save, sender=RollingStock) def pre_save_running_number(sender, instance, *args, **kwargs): @@ -126,10 +137,25 @@ class RollingStockDocument(Document): unique_together = ("rolling_stock", "file") +def rolling_stock_image_upload(instance, filename): + return os.path.join( + "images", + "rollingstock", + str(instance.rolling_stock.uuid), + filename + ) + + class RollingStockImage(Image): rolling_stock = models.ForeignKey( RollingStock, on_delete=models.CASCADE, related_name="image" ) + image = models.ImageField( + upload_to=rolling_stock_image_upload, + storage=DeduplicatedStorage, + null=True, + blank=True, + ) class RollingStockProperty(PropertyInstance):