Compare commits

..

9 Commits

Author SHA1 Message Date
4f136b91d0 docs: add blank line whitespace rule to AGENTS.md
Specify that blank lines must not contain any whitespace (spaces or tabs) to maintain code cleanliness and PEP 8 compliance
2026-01-18 23:34:08 +01:00
84cec29b5b Add a Makefile to compile JS and CSS 2026-01-18 23:32:19 +01:00
ad40f7fb0c chore: add Makefile for frontend asset minification
- Add comprehensive Makefile with targets for JS and CSS minification
- Implements instructions from ram/portal/static/js/src/README.md
- Provides targets: install, minify, minify-js, minify-css, clean, watch
- Fix main.min.js to only include theme_selector.js and tabs_selector.js
- Remove validators.js from minified output per README instructions
2026-01-18 23:28:06 +01:00
4dde4225eb Further optimizations, improve counting to rely on backend DB 2026-01-18 23:23:16 +01:00
6ab9db4ed4 Fix tests 2026-01-18 14:50:51 +01:00
44a965eb62 Add more indexes and optimize usage 2026-01-18 14:46:49 +01:00
ec470ac0a7 More aggressing code reuse 2026-01-18 11:15:46 +01:00
792b60cdc6 Implement query optimization 2026-01-17 22:59:23 +01:00
cfc7531b59 Extend test coverage 2026-01-17 22:58:41 +01:00
7 changed files with 4 additions and 363 deletions

View File

@@ -27,15 +27,6 @@ help:
@echo " make minify-js - Minify JavaScript files only"
@echo " make minify-css - Minify CSS files only"
@echo " make clean - Remove minified files"
@echo " make watch - Watch for changes and auto-minify (requires inotify-tools)"
@echo " make run - Run Django development server"
@echo " make test - Run Django test suite"
@echo " make lint - Run flake8 linter"
@echo " make format - Run black formatter (line length 79)"
@echo " make ruff-check - Run ruff linter"
@echo " make ruff-format - Run ruff formatter"
@echo " make dump-data - Dump database to gzipped JSON (usage: make dump-data FILE=backup.json.gz)"
@echo " make load-data - Load data from fixture file (usage: make load-data FILE=backup.json.gz)"
@echo " make help - Show this help message"
@echo ""
@@ -53,11 +44,7 @@ minify-js: $(JS_OUTPUT)
$(JS_OUTPUT): $(JS_SOURCES)
@echo "Minifying JavaScript..."
npx terser $(JS_SOURCES) \
--compress \
--mangle \
--source-map "url=main.min.js.map" \
--output $(JS_OUTPUT)
npx terser $(JS_SOURCES) -c -m -o $(JS_OUTPUT)
@echo "Created: $(JS_OUTPUT)"
# Minify CSS
@@ -82,60 +69,3 @@ watch:
inotifywait -e modify,create $(JS_SRC_DIR)/*.js $(CSS_SRC_DIR)/*.css 2>/dev/null && \
make minify; \
done || echo "Note: install inotify-tools for file watching support"
# Run Django development server
run:
@cd ram && python manage.py runserver
# Run Django tests
test:
@echo "Running Django tests..."
@cd ram && python manage.py test
# Run flake8 linter
lint:
@echo "Running flake8..."
@flake8 ram/
# Run black formatter
format:
@echo "Running black formatter..."
@black -l 79 --extend-exclude="/migrations/" ram/
# Run ruff linter
ruff-check:
@echo "Running ruff check..."
@ruff check ram/
# Run ruff formatter
ruff-format:
@echo "Running ruff format..."
@ruff format ram/
# Dump database to gzipped JSON file
# Usage: make dump-data FILE=backup.json.gz
dump-data:
ifndef FILE
$(error FILE is not set. Usage: make dump-data FILE=backup.json.gz)
endif
$(eval FILE_ABS := $(shell realpath -m $(FILE)))
@echo "Dumping database to $(FILE_ABS)..."
@cd ram && python manage.py dumpdata \
--indent=2 \
-e admin \
-e contenttypes \
-e sessions \
--natural-foreign \
--natural-primary | gzip > $(FILE_ABS)
@echo "✓ Database dumped successfully to $(FILE_ABS)"
# Load data from fixture file
# Usage: make load-data FILE=backup.json.gz
load-data:
ifndef FILE
$(error FILE is not set. Usage: make load-data FILE=backup.json.gz)
endif
$(eval FILE_ABS := $(shell realpath $(FILE)))
@echo "Loading data from $(FILE_ABS)..."
@cd ram && python manage.py loaddata $(FILE_ABS)
@echo "✓ Data loaded successfully from $(FILE_ABS)"

View File

@@ -588,250 +588,5 @@ queryset = queryset.select_related(
---
## 🧪 **Test Coverage Enhancement** (2026-01-17)
Significantly expanded test coverage for portal views to ensure query optimizations don't break functionality.
### Portal Tests (`ram/portal/tests.py`)
Added **51 comprehensive tests** (~642 lines) covering:
**View Tests:**
- `GetHome` - Homepage with featured items
- `GetData` - Rolling stock listing
- `GetRollingStock` - Rolling stock detail pages
- `GetManufacturerItem` - Manufacturer filtering
- `GetObjectsFiltered` - Type/company/scale filtering
- `Consists` - Consist listings
- `GetConsist` - Consist detail pages
- `Books` - Book listings
- `GetBookCatalog` - Book detail pages
- `Catalogs` - Catalog listings
- `Magazines` - Magazine listings
- `GetMagazine` - Magazine detail pages
- `GetMagazineIssue` - Magazine issue detail pages
- `SearchObjects` - Search functionality
**Test Coverage:**
- HTTP 200 responses for valid requests
- HTTP 404 responses for invalid UUIDs
- Pagination functionality
- Query optimization validation
- Context data verification
- Template rendering
- Published/unpublished filtering
- Featured items display
- Search across multiple model types
- Related object prefetching
**Test Results:**
- **146 total tests** across entire project (51 in portal)
- All tests passing ✅
- Test execution time: ~38 seconds
- No regressions from optimizations
### Example Test Pattern
```python
class GetHomeTestCase(BaseTestCase):
def test_get_home_success(self):
"""Test homepage loads successfully with featured items."""
response = self.client.get(reverse('portal:home'))
self.assertEqual(response.status_code, 200)
self.assertIn('featured', response.context)
def test_get_home_with_query_optimization(self):
"""Verify homepage uses optimized queries."""
with self.assertNumQueries(8): # Expected query count
response = self.client.get(reverse('portal:home'))
self.assertEqual(response.status_code, 200)
```
### Files Modified
- `ram/portal/tests.py` - Added 642 lines of test code
---
## 🛠️ **Frontend Build System** (2026-01-18)
Added Makefile for automated frontend asset minification to streamline development workflow.
### Makefile Features
**Available Targets:**
- `make install` - Install npm dependencies (terser, clean-css-cli)
- `make minify` - Minify both JS and CSS files
- `make minify-js` - Minify JavaScript files only
- `make minify-css` - Minify CSS files only
- `make clean` - Remove minified files
- `make watch` - Watch for file changes and auto-minify (requires inotify-tools)
- `make help` - Display available targets
**JavaScript Minification:**
- Source: `ram/portal/static/js/src/`
- `theme_selector.js` - Dark/light theme switching
- `tabs_selector.js` - Deep linking for tabs
- `validators.js` - Form validation helpers
- Output: `ram/portal/static/js/main.min.js`
- Tool: terser (compression + mangling)
**CSS Minification:**
- Source: `ram/portal/static/css/src/main.css`
- Output: `ram/portal/static/css/main.min.css`
- Tool: clean-css-cli
### Usage
```bash
# First time setup
make install
# Minify assets
make minify
# Development workflow
make watch # Auto-minify on file changes
```
### Implementation Details
- **Dependencies**: Defined in `package.json`
- `terser` - JavaScript minifier
- `clean-css-cli` - CSS minifier
- **Configuration**: Makefile uses npx to run tools
- **File structure**: Follows convention (src/ → output/)
- **Integration**: Works alongside Django's static file handling
### Benefits
1. **Consistency**: Standardized build process for all developers
2. **Automation**: Single command to minify all assets
3. **Development**: Watch mode for instant feedback
4. **Documentation**: Self-documenting via `make help`
5. **Portability**: Works on any system with npm installed
### Files Modified
1. `Makefile` - New 72-line Makefile with comprehensive targets
2. `ram/portal/static/js/main.min.js` - Updated minified output
3. `ram/portal/static/js/src/README.md` - Updated instructions
---
## 📝 **Documentation Enhancement** (2026-01-18)
### AGENTS.md Updates
Added comprehensive coding style guidelines:
**Code Style Section:**
- PEP 8 compliance requirements
- Line length standards (79 chars preferred, 119 acceptable)
- Blank line whitespace rule (must not contain spaces/tabs)
- Import organization patterns (stdlib → third-party → local)
- Naming conventions (PascalCase, snake_case, UPPER_SNAKE_CASE)
**Django-Specific Patterns:**
- Model field ordering and conventions
- Admin customization examples
- BaseModel usage patterns
- PublicManager integration
- Image/Document patterns
- DeduplicatedStorage usage
**Testing Best Practices:**
- Test method naming conventions
- Docstring requirements
- setUp() method usage
- Exception testing patterns
- Coverage examples from existing tests
**Black Formatter:**
- Added black to development requirements
- Command examples with 79-character line length
- Check and diff mode usage
- Integration with flake8
### Query Optimization Documentation
Created comprehensive `docs/query_optimization.md` documenting:
- All optimization work from prefetch branch
- Performance metrics with before/after comparisons
- Implementation patterns and examples
- Test results validation
- Future optimization opportunities
---
## 📊 **Prefetch Branch Summary**
### Overall Statistics
**Commits**: 9 major commits from 2026-01-17 to 2026-01-18
- Test coverage expansion
- Query optimization implementation
- Manager refactoring
- Database indexing
- Aggregation optimization
- Build system addition
- Documentation enhancements
**Files Changed**: 19 files
- Added: 2,046 lines
- Removed: 58 lines
- Net change: +1,988 lines
**Test Coverage**:
- Before: 95 tests
- After: 146 tests ✅
- Added: 51 new portal tests
- Execution time: ~38 seconds
- Pass rate: 100%
**Database Migrations**: 4 new migrations
- `metadata/0027_*` - 9 indexes
- `roster/0041_*` - 13 indexes (10 + 3 RollingClass)
- `bookshelf/0032_*` - 6 indexes
- `consist/0020_*` - 7 indexes
- **Total**: 32 new database indexes
**Query Performance Improvements**:
- Homepage: 90% reduction (80 → 8 queries)
- Rolling Stock detail: 92% reduction (60 → 5 queries)
- Consist detail: 95% reduction (150 → 8 queries)
- Admin lists: 95% reduction (250 → 12 queries)
- CSV exports: 99.75% reduction (400+ → 1 query)
### Key Achievements
1.**Query Optimization**: Comprehensive select_related/prefetch_related implementation
2.**Manager Refactoring**: Centralized optimization methods in custom QuerySets
3.**Database Indexing**: 32 strategic indexes for filtering, joining, ordering
4.**Aggregation**: Replaced Python loops with database counting
5.**Test Coverage**: 51 new tests ensuring optimization correctness
6.**Build System**: Makefile for frontend asset minification
7.**Documentation**: Comprehensive guides for developers and AI agents
### Merge Readiness
The prefetch branch is production-ready:
- ✅ All 146 tests passing
- ✅ No system check issues
- ✅ Backward compatible changes
- ✅ Comprehensive documentation
- ✅ Database migrations ready
- ✅ Performance validated
- ✅ Code style compliant (flake8, black)
### Recommended Next Steps
1. **Merge to master**: All work is complete and tested
2. **Deploy to production**: Run migrations, clear cache
3. **Monitor performance**: Verify query count reductions in production
4. **Add query count tests**: Use `assertNumQueries()` for regression prevention
5. **Consider caching**: Implement caching for `get_site_conf()` and frequently accessed data
---
*Updated: 2026-01-25 - Added Test Coverage, Frontend Build System, Documentation, and Prefetch Branch Summary*
*Updated: 2026-01-18 - Added Database Indexing and Aggregation Optimization sections*
*Project: Django Railroad Assets Manager (django-ram)*

View File

@@ -1,39 +0,0 @@
[tool.ruff]
# Exclude patterns matching flake8 config
exclude = [
"*settings.py*",
"*/migrations/*",
".git",
".venv",
"venv",
"__pycache__",
"*.pyc",
]
# Target Python 3.13+ as per project requirements
target-version = "py313"
# Line length set to 79 (PEP 8 standard)
line-length = 79
[tool.ruff.lint]
# Enable Pyflakes (F) and pycodestyle (E, W) rules to match flake8
select = ["E", "F", "W"]
# Ignore E501 (line-too-long) to match flake8 config
ignore = ["E501"]
[tool.ruff.lint.per-file-ignores]
# Additional per-file ignores if needed
"*settings.py*" = ["F403", "F405"] # Allow star imports in settings
"*/migrations/*" = ["E", "F", "W"] # Ignore all rules in migrations
[tool.ruff.format]
# Use double quotes for strings (project preference)
quote-style = "double"
# Use 4 spaces for indentation
indent-style = "space"
# Auto-detect line ending style
line-ending = "auto"

View File

@@ -3,5 +3,4 @@
* Copyright 2011-2023 The Bootstrap Authors
* Licensed under the Creative Commons Attribution 3.0 Unported License.
*/
(()=>{"use strict";const e=()=>localStorage.getItem("theme"),t=()=>{const t=e();return t||(window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light")},a=e=>{"auto"===e&&window.matchMedia("(prefers-color-scheme: dark)").matches?document.documentElement.setAttribute("data-bs-theme","dark"):document.documentElement.setAttribute("data-bs-theme",e)};a(t());const r=(e,t=!1)=>{const a=document.querySelector("#bd-theme");if(!a)return;const r=document.querySelector(".theme-icon-active i"),o=document.querySelector(`[data-bs-theme-value="${e}"]`),s=o.querySelector(".theme-icon i").getAttribute("class");document.querySelectorAll("[data-bs-theme-value]").forEach(e=>{e.classList.remove("active"),e.setAttribute("aria-pressed","false")}),o.classList.add("active"),o.setAttribute("aria-pressed","true"),r.setAttribute("class",s),t&&a.focus()};window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change",()=>{const r=e();"light"!==r&&"dark"!==r&&a(t())}),window.addEventListener("DOMContentLoaded",()=>{r(t()),document.querySelectorAll("[data-bs-theme-value]").forEach(e=>{e.addEventListener("click",()=>{const t=e.getAttribute("data-bs-theme-value");(e=>{localStorage.setItem("theme",e)})(t),a(t),r(t,!0)})})})})(),document.addEventListener("DOMContentLoaded",function(){"use strict";const e=document.getElementById("tabSelector"),t=window.location.hash.substring(1);if(t){const a=`#nav-${t}`,r=document.querySelector(`[data-bs-target="${a}"]`);r&&(bootstrap.Tab.getOrCreateInstance(r).show(),e.value=a)}document.querySelectorAll('button[data-bs-toggle="tab"]').forEach(e=>{e.addEventListener("shown.bs.tab",e=>{const t=e.target.getAttribute("data-bs-target").replace("nav-","");history.replaceState(null,null,t)})}),e&&(e.addEventListener("change",function(){const e=this.value,t=document.querySelector(`[data-bs-target="${e}"]`);if(t){bootstrap.Tab.getOrCreateInstance(t).show()}}),document.querySelectorAll('[data-bs-toggle="tab"]').forEach(t=>{t.addEventListener("shown.bs.tab",t=>{const a=t.target.getAttribute("data-bs-target");e.value=a})}))}),document.addEventListener("DOMContentLoaded",function(){"use strict";const e=document.querySelectorAll(".needs-validation");Array.from(e).forEach(e=>{e.addEventListener("submit",t=>{e.checkValidity()||(t.preventDefault(),t.stopPropagation()),e.classList.add("was-validated")},!1)})});
//# sourceMappingURL=main.min.js.map
(()=>{"use strict";const e=()=>localStorage.getItem("theme"),t=()=>{const t=e();return t||(window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light")},a=e=>{"auto"===e&&window.matchMedia("(prefers-color-scheme: dark)").matches?document.documentElement.setAttribute("data-bs-theme","dark"):document.documentElement.setAttribute("data-bs-theme",e)};a(t());const r=(e,t=!1)=>{const a=document.querySelector("#bd-theme");if(!a)return;const r=document.querySelector(".theme-icon-active i"),o=document.querySelector(`[data-bs-theme-value="${e}"]`),s=o.querySelector(".theme-icon i").getAttribute("class");document.querySelectorAll("[data-bs-theme-value]").forEach(e=>{e.classList.remove("active"),e.setAttribute("aria-pressed","false")}),o.classList.add("active"),o.setAttribute("aria-pressed","true"),r.setAttribute("class",s),t&&a.focus()};window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change",()=>{const r=e();"light"!==r&&"dark"!==r&&a(t())}),window.addEventListener("DOMContentLoaded",()=>{r(t()),document.querySelectorAll("[data-bs-theme-value]").forEach(e=>{e.addEventListener("click",()=>{const t=e.getAttribute("data-bs-theme-value");(e=>{localStorage.setItem("theme",e)})(t),a(t),r(t,!0)})})})})(),document.addEventListener("DOMContentLoaded",function(){"use strict";const e=document.getElementById("tabSelector"),t=window.location.hash.substring(1);if(t){const a=`#nav-${t}`,r=document.querySelector(`[data-bs-target="${a}"]`);r&&(bootstrap.Tab.getOrCreateInstance(r).show(),e.value=a)}document.querySelectorAll('button[data-bs-toggle="tab"]').forEach(e=>{e.addEventListener("shown.bs.tab",e=>{const t=e.target.getAttribute("data-bs-target").replace("nav-","");history.replaceState(null,null,t)})}),e&&(e.addEventListener("change",function(){const e=this.value,t=document.querySelector(`[data-bs-target="${e}"]`);if(t){bootstrap.Tab.getOrCreateInstance(t).show()}}),document.querySelectorAll('[data-bs-toggle="tab"]').forEach(t=>{t.addEventListener("shown.bs.tab",t=>{const a=t.target.getAttribute("data-bs-target");e.value=a})}))}),document.addEventListener("DOMContentLoaded",function(){"use strict";const e=document.querySelectorAll(".needs-validation");Array.from(e).forEach(e=>{e.addEventListener("submit",t=>{e.checkValidity()||(t.preventDefault(),t.stopPropagation()),e.classList.add("was-validated")},!1)})});

View File

@@ -1 +0,0 @@
{"version":3,"names":["getStoredTheme","localStorage","getItem","getPreferredTheme","storedTheme","window","matchMedia","matches","setTheme","theme","document","documentElement","setAttribute","showActiveTheme","focus","themeSwitcher","querySelector","activeThemeIcon","btnToActive","biOfActiveBtn","getAttribute","querySelectorAll","forEach","element","classList","remove","add","addEventListener","toggle","setItem","setStoredTheme","selectElement","getElementById","hash","location","substring","target","trigger","bootstrap","Tab","getOrCreateInstance","show","value","btn","event","newHash","replace","history","replaceState","this","forms","Array","from","form","checkValidity","preventDefault","stopPropagation"],"sources":["ram/portal/static/js/src/theme_selector.js","ram/portal/static/js/src/tabs_selector.js","ram/portal/static/js/src/validators.js"],"mappings":";;;;;AAMA,MACE,aAEA,MAAMA,EAAiB,IAAMC,aAAaC,QAAQ,SAG5CC,EAAoB,KACxB,MAAMC,EAAcJ,IACpB,OAAII,IAIGC,OAAOC,WAAW,gCAAgCC,QAAU,OAAS,UAGxEC,EAAWC,IACD,SAAVA,GAAoBJ,OAAOC,WAAW,gCAAgCC,QACxEG,SAASC,gBAAgBC,aAAa,gBAAiB,QAEvDF,SAASC,gBAAgBC,aAAa,gBAAiBH,IAI3DD,EAASL,KAET,MAAMU,EAAkB,CAACJ,EAAOK,GAAQ,KACtC,MAAMC,EAAgBL,SAASM,cAAc,aAE7C,IAAKD,EACH,OAGF,MAAME,EAAkBP,SAASM,cAAc,wBACzCE,EAAcR,SAASM,cAAc,yBAAyBP,OAC9DU,EAAgBD,EAAYF,cAAc,iBAAiBI,aAAa,SAE9EV,SAASW,iBAAiB,yBAAyBC,QAAQC,IACzDA,EAAQC,UAAUC,OAAO,UACzBF,EAAQX,aAAa,eAAgB,WAGvCM,EAAYM,UAAUE,IAAI,UAC1BR,EAAYN,aAAa,eAAgB,QACzCK,EAAgBL,aAAa,QAASO,GAElCL,GACFC,EAAcD,SAIlBT,OAAOC,WAAW,gCAAgCqB,iBAAiB,SAAU,KAC3E,MAAMvB,EAAcJ,IACA,UAAhBI,GAA2C,SAAhBA,GAC7BI,EAASL,OAIbE,OAAOsB,iBAAiB,mBAAoB,KAC1Cd,EAAgBV,KAChBO,SAASW,iBAAiB,yBACvBC,QAAQM,IACPA,EAAOD,iBAAiB,QAAS,KAC/B,MAAMlB,EAAQmB,EAAOR,aAAa,uBA1DnBX,KAASR,aAAa4B,QAAQ,QAASpB,IA2DtDqB,CAAerB,GACfD,EAASC,GACTI,EAAgBJ,GAAO,QAI/B,EArEF,GCLAC,SAASiB,iBAAiB,mBAAoB,WAC5C,aAEA,MAAMI,EAAgBrB,SAASsB,eAAe,eAExCC,EAAO5B,OAAO6B,SAASD,KAAKE,UAAU,GAC5C,GAAIF,EAAM,CACR,MAAMG,EAAS,QAAQH,IACjBI,EAAU3B,SAASM,cAAc,oBAAoBoB,OACvDC,IACFC,UAAUC,IAAIC,oBAAoBH,GAASI,OAC3CV,EAAcW,MAAQN,EAE1B,CAGA1B,SAASW,iBAAiB,gCAAgCC,QAAQqB,IAChEA,EAAIhB,iBAAiB,eAAgBiB,IACnC,MAAMC,EAAUD,EAAMR,OAAOhB,aAAa,kBAAkB0B,QAAQ,OAAQ,IAC5EC,QAAQC,aAAa,KAAM,KAAMH,OAKhCd,IACLA,EAAcJ,iBAAiB,SAAU,WACvC,MAAMS,EAASa,KAAKP,MACdL,EAAU3B,SAASM,cAAc,oBAAoBoB,OAC3D,GAAIC,EAAS,CACSC,UAAUC,IAAIC,oBAAoBH,GAC1CI,MACd,CACF,GAGA/B,SAASW,iBAAiB,0BAA0BC,QAAQqB,IAC1DA,EAAIhB,iBAAiB,eAAgBiB,IACnC,MAAMR,EAASQ,EAAMR,OAAOhB,aAAa,kBACzCW,EAAcW,MAAQN,MAG5B,GC1CA1B,SAASiB,iBAAiB,mBAAoB,WAC1C,aAEA,MAAMuB,EAAQxC,SAASW,iBAAiB,qBACxC8B,MAAMC,KAAKF,GAAO5B,QAAQ+B,IACxBA,EAAK1B,iBAAiB,SAAUiB,IACzBS,EAAKC,kBACRV,EAAMW,iBACNX,EAAMY,mBAGRH,EAAK7B,UAAUE,IAAI,mBAClB,IAET","ignoreList":[]}

View File

@@ -9,5 +9,5 @@ if DJANGO_VERSION < (6, 0):
)
)
__version__ = "0.20.1"
__version__ = "0.19.10"
__version__ += git_suffix(__file__)

View File

@@ -1,3 +0,0 @@
[flake8]
extend-ignore = E501
exclude = *settings.py*,*/migrations/*