dockerize app + add celery + impl pushpin for message events

.gitignore

@@ -2,3 +2,6 @@
 __pycache__
 db.sqlite3
 venv
+static
+*.log
+*.pid

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.10
+
+ENV PYTHONUNBUFFERED 1
+ENV DJANGO_SETTINGS_MODULE mumui.settings
+
+RUN mkdir /app
+WORKDIR /app
+
+RUN apt update && apt install -y
+
+COPY startup.sh /app/
+COPY requirements.txt /app/
+
+RUN pip install -r requirements.txt
+
+ENTRYPOINT [ "/app/startup.sh" ]
+
+EXPOSE 8000

Makefile

@@ -1,4 +1,4 @@
-#TODO(rmanach): add a pyproject.toml
+# TODO(rmanach): add a pyproject.toml
 
 format:
 	black deployment/*.py
@@ -12,5 +12,17 @@ migrations:
 	python manage.py makemigrations
 	python manage.py migrate
 
+django:
+	docker build . -t mumui:local
+
+pushpin-local:
+	cd pushpin && docker build . -t pushpin:local
+
+nginx-local:
+	cd nginx && docker build . -t nginx:local
+
 run:
-	python manage.py runserver
+	docker compose up
+
+stop:
+	docker compose down

deployment/migrations/0001_initial.py

@@ -1,9 +1,8 @@
-# Generated by Django 4.2 on 2023-09-15 08:14
+# Generated by Django 4.2.5 on 2023-09-18 08:14
 
 from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import uuid
 
 
 class Migration(migrations.Migration):
@@ -17,21 +16,32 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name="Deployment",
             fields=[
-                (
+                ("id", models.UUIDField(primary_key=True, serialize=False)),
-                    "id",
-                    models.UUIDField(
-                        default=uuid.UUID("d7e9833f-e912-4640-8bf0-19e114a1136f"),
-                        primary_key=True,
-                        serialize=False,
-                    ),
-                ),
                 ("name", models.TextField()),
                 (
                     "type",
                     models.CharField(
-                        choices=[("S", "Light"), ("M", "Medium"), ("L", "Heavy")],
+                        choices=[
-                        default="S",
+                            ("SLIM", "Slim"),
-                        max_length=1,
+                            ("MEDIUM", "Medium"),
+                            ("LARGE", "Large"),
+                        ],
+                        default="SLIM",
+                        max_length=6,
+                    ),
+                ),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("READY", "Ready"),
+                            ("PENDING", "Pending"),
+                            ("RUNNING", "Running"),
+                            ("SUCCESS", "Success"),
+                            ("FAILED", "Failed"),
+                        ],
+                        default="READY",
+                        max_length=7,
                     ),
                 ),
                 (

deployment/migrations/0002_deployment_status_alter_deployment_id.py

@@ -1,36 +0,0 @@
-# Generated by Django 4.2 on 2023-09-15 08:31
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("deployment", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="deployment",
-            name="status",
-            field=models.CharField(
-                choices=[
-                    ("PENDING", "Pending"),
-                    ("RUNNING", "Running"),
-                    ("SUCCESS", "Success"),
-                    ("FAILED", "Failed"),
-                ],
-                default="PENDING",
-                max_length=7,
-            ),
-        ),
-        migrations.AlterField(
-            model_name="deployment",
-            name="id",
-            field=models.UUIDField(
-                default=uuid.UUID("7909f7f2-0ee4-45f6-b8d4-beb5df8efc73"),
-                primary_key=True,
-                serialize=False,
-            ),
-        ),
-    ]

deployment/migrations/0003_alter_deployment_id_alter_deployment_status.py

@@ -1,24 +0,0 @@
-# Generated by Django 4.2.5 on 2023-09-16 08:16
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('deployment', '0002_deployment_status_alter_deployment_id'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='deployment',
-            name='id',
-            field=models.UUIDField(default=uuid.UUID('8584555b-51ea-46a4-aebc-f7a8ab8670a4'), primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='deployment',
-            name='status',
-            field=models.CharField(choices=[('READY', 'Ready'), ('PENDING', 'Pending'), ('RUNNING', 'Running'), ('SUCCESS', 'Success'), ('FAILED', 'Failed')], default='READY', max_length=7),
-        ),
-    ]

deployment/migrations/0004_alter_deployment_id_alter_deployment_status_and_more.py

@@ -1,47 +0,0 @@
-# Generated by Django 4.2.5 on 2023-09-16 08:43
-
-import deployment.models
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("deployment", "0003_alter_deployment_id_alter_deployment_status"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="deployment",
-            name="id",
-            field=models.UUIDField(
-                default=uuid.UUID("d29e958f-7b3d-43a8-bc17-a40ab4184dc6"),
-                primary_key=True,
-                serialize=False,
-            ),
-        ),
-        migrations.AlterField(
-            model_name="deployment",
-            name="status",
-            field=models.CharField(
-                choices=[
-                    ("READY", "Ready"),
-                    ("PENDING", "Pending"),
-                    ("RUNNING", "Running"),
-                    ("SUCCESS", "Success"),
-                    ("FAILED", "Failed"),
-                ],
-                default=deployment.models.Status["READY"],
-                max_length=7,
-            ),
-        ),
-        migrations.AlterField(
-            model_name="deployment",
-            name="type",
-            field=models.CharField(
-                choices=[("SLIM", "Slim"), ("MEDIUM", "Medium"), ("LARGE", "Large")],
-                default=deployment.models.Type["SLIM"],
-                max_length=6,
-            ),
-        ),
-    ]

deployment/migrations/0005_alter_deployment_id.py

@@ -1,22 +0,0 @@
-# Generated by Django 4.2.5 on 2023-09-16 08:43
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("deployment", "0004_alter_deployment_id_alter_deployment_status_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="deployment",
-            name="id",
-            field=models.UUIDField(
-                default=uuid.UUID("ec7ebcb5-420a-47c3-ba07-e8c340895865"),
-                primary_key=True,
-                serialize=False,
-            ),
-        ),
-    ]

deployment/migrations/0006_alter_deployment_id_alter_deployment_status_and_more.py

@@ -1,46 +0,0 @@
-# Generated by Django 4.2.5 on 2023-09-16 08:45
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("deployment", "0005_alter_deployment_id"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="deployment",
-            name="id",
-            field=models.UUIDField(
-                default=uuid.UUID("b8feb7aa-9c79-478b-ba91-df2e03f4145b"),
-                primary_key=True,
-                serialize=False,
-            ),
-        ),
-        migrations.AlterField(
-            model_name="deployment",
-            name="status",
-            field=models.CharField(
-                choices=[
-                    ("READY", "Ready"),
-                    ("PENDING", "Pending"),
-                    ("RUNNING", "Running"),
-                    ("SUCCESS", "Success"),
-                    ("FAILED", "Failed"),
-                ],
-                default="Ready",
-                max_length=7,
-            ),
-        ),
-        migrations.AlterField(
-            model_name="deployment",
-            name="type",
-            field=models.CharField(
-                choices=[("SLIM", "Slim"), ("MEDIUM", "Medium"), ("LARGE", "Large")],
-                default="Slim",
-                max_length=6,
-            ),
-        ),
-    ]

deployment/migrations/0007_alter_deployment_id.py

@@ -1,22 +0,0 @@
-# Generated by Django 4.2.5 on 2023-09-16 17:21
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("deployment", "0006_alter_deployment_id_alter_deployment_status_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="deployment",
-            name="id",
-            field=models.UUIDField(
-                default=uuid.UUID("0cc4ae03-f52e-4f3a-9fe1-fecda707b20e"),
-                primary_key=True,
-                serialize=False,
-            ),
-        ),
-    ]

deployment/models.py

@@ -4,9 +4,6 @@ from uuid import uuid4
 
 from django.contrib.auth.models import User
 from django.db import models
-from django.db.models.signals import pre_save, pre_delete
-from django.dispatch import receiver
-from django_eventstream import send_event
 
 
 class Type(Enum):
@@ -32,21 +29,15 @@ class Status(Enum):
 
 
 class Deployment(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid4())
+    id = models.UUIDField(primary_key=True)
     name = models.TextField(null=False, blank=False)
     type = models.CharField(
-        max_length=6, choices=Type.into_choices(), default=Type.SLIM.value
+        max_length=6, choices=Type.into_choices(), default=Type.SLIM.name
     )
     status = models.CharField(
-        max_length=7, choices=Status.into_choices(), default=Status.READY.value
+        max_length=7, choices=Status.into_choices(), default=Status.READY.name
     )
     user = models.ForeignKey(User, on_delete=models.CASCADE)
 
     def __str__(self):
         return f"{self.name} | {self.type} | {self.status}"
-
-
-@receiver(pre_save, sender=Deployment)
-@receiver(pre_delete, sender=Deployment)
-def deployment_update_handler(sender, instance, **kwargs):
-    send_event("deployment", "update", {"id": instance.id, "status": instance.status})

deployment/tasks.py

@@ -0,0 +1,32 @@
+import time
+import random
+from uuid import UUID
+
+from celery import shared_task
+from django_eventstream import send_event
+
+from deployment.models import Deployment, Type, Status
+
+
+@shared_task
+def deploy(deployment_id: UUID):
+    deploy = Deployment.objects.get(id=deployment_id)
+
+    deploy.status = Status.RUNNING.name
+    deploy.save()
+
+    match deploy.type:
+        case Type.SLIM.name:
+            time.sleep(10)
+        case Type.MEDIUM.name:
+            time.sleep(60)
+        case Type.LARGE.name:
+            time.sleep(120)
+
+    deploy.status = (
+        Status.FAILED.name if random.randint(0, 10) % 2 != 0 else Status.SUCCESS.name
+    )
+    deploy.save()
+
+    send_event("test", "message", {"id": deploy.id, "status": deploy.status})
+    print("event sent")

deployment/urls.py

@@ -1,8 +1,12 @@
-from django.urls import path
+import django_eventstream
-from deployment.views import index, create, details
+
+from django.urls import path, include
+from deployment.views import index, create, details, deploy
 
 urlpatterns = [
     path("", index, name="deployment"),
     path("create", create, name="deployment-create"),
     path("<uuid:deployment_id>", details, name="deployment-details"),
+    path("<uuid:deployment_id>/deploy", deploy, name="deployment-launch"),
+    path("events/", include(django_eventstream.urls), {"channels": ["test"]}),
 ]

deployment/views.py

@@ -10,6 +10,7 @@ from django.shortcuts import render, get_object_or_404
 
 from deployment.forms import DeploymentForm
 from deployment.models import Deployment, Status
+from deployment.tasks import deploy as launch_deploy
 
 
 def index(request):
@@ -20,10 +21,24 @@ def index(request):
     page_obj = paginator.get_page(page_number)
 
     return render(
-        request, "deployment/board.html", {"page_obj": page_obj, "url": "/events/"}
+        request, "deployment/board.html", {"page_obj": page_obj, "url": "events/"}
     )
 
 
+def deploy(request, deployment_id):
+    deployment = get_object_or_404(Deployment, id=deployment_id)
+
+    if request.method == "POST":
+        deployment.status = Status.PENDING.name
+        deployment.save()
+        launch_deploy.delay(deployment_id)
+
+    if page := request.GET.get("page", ""):
+        return HttpResponseRedirect(f"/deployment?page={page}")
+
+    return HttpResponseRedirect("/deployment")
+
+
 def details(request, deployment_id):
     deployment = get_object_or_404(Deployment, id=deployment_id)
 

docker-compose.yml

@@ -0,0 +1,60 @@
+version: '3'
+services:
+  redis:
+    image: redis/redis-stack-server:latest
+    container_name: redis
+    networks:
+      - mumui_network
+    volumes:
+      - redis_data:/data
+
+  pushpin:
+    image: pushpin:local
+    container_name: pushpin
+    networks:
+      - mumui_network
+    depends_on:
+      - mumui
+
+  postgres:
+    image: postgres:latest
+    container_name: postgres
+    environment:
+      POSTGRES_DB: mumui
+      POSTGRES_USER: test
+      POSTGRES_PASSWORD: test
+    networks:
+      - mumui_network
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+  mumui:
+    image: mumui:local
+    container_name: mumui
+    networks:
+      - mumui_network
+    volumes:
+      - .:/app
+    depends_on:
+      - postgres
+
+  nginx:
+    image: nginx:local
+    container_name: nginx
+    networks:
+      - mumui_network
+    volumes:
+      - ./static:/app/static
+    ports:
+      - "8080:8080"
+    depends_on:
+      - mumui
+  
+
+networks:
+  mumui_network:
+    driver: bridge
+
+volumes:
+  redis_data:
+  postgres_data:

mumui/__init__.py

@@ -0,0 +1,3 @@
+from mumui.celery import app as celery_app
+
+__all__ = ("celery_app",)

mumui/asgi.py

@@ -7,28 +7,28 @@ For more information on this file, see
 https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
 """
 
-import os
+# import os
-from django.core.asgi import get_asgi_application
+# from django.core.asgi import get_asgi_application
-from django.urls import path, re_path
+# from django.urls import path, re_path
-from channels.routing import ProtocolTypeRouter, URLRouter
+# from channels.routing import ProtocolTypeRouter, URLRouter
-from channels.auth import AuthMiddlewareStack
+# from channels.auth import AuthMiddlewareStack
-import django_eventstream
+# import django_eventstream
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings")
+# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings")
 
-application = ProtocolTypeRouter(
+# application = ProtocolTypeRouter(
-    {
+#     {
-        "http": URLRouter(
+#         "http": URLRouter(
-            [
+#             [
-                path(
+#                 path(
-                    "events/",
+#                     "events/",
-                    AuthMiddlewareStack(
+#                     AuthMiddlewareStack(
-                        URLRouter(django_eventstream.routing.urlpatterns)
+#                         URLRouter(django_eventstream.routing.urlpatterns)
-                    ),
+#                     ),
-                    {"channels": ["test"]},
+#                     {"channels": ["deployment"]},
-                ),
+#                 ),
-                re_path(r"", get_asgi_application()),
+#                 re_path(r"", get_asgi_application()),
-            ]
+#             ]
-        ),
+#         ),
-    }
+#     }
-)
+# )

mumui/celery.py

@@ -0,0 +1,10 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mumui.settings")
+
+app = Celery("mumui", broker="redis://redis:6379/0")
+
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()

mumui/settings.py

@@ -25,7 +25,7 @@ SECRET_KEY = "django-insecure-_c56%%c8%g%@5(3&thxi7ku2a&wst8lik*8@l0=#)ar)s86g36
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = []
+ALLOWED_HOSTS = ["*"]
 
 
 # Application definition
@@ -38,7 +38,6 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "deployment",
-    "channels",
     "django_eventstream",
 ]
 
@@ -79,12 +78,15 @@ WSGI_APPLICATION = "mumui.wsgi.application"
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.sqlite3",
+        "ENGINE": "django.db.backends.postgresql",
-        "NAME": BASE_DIR / "db.sqlite3",
+        "NAME": "mumui",
+        "USER": "test",
+        "PASSWORD": "test",
+        "HOST": "postgres",
+        "PORT": "5432",
     }
 }
 
-
 # Password validation
 # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
 
@@ -120,6 +122,7 @@ USE_TZ = True
 # https://docs.djangoproject.com/en/4.2/howto/static-files/
 
 STATIC_URL = "static/"
+STATIC_ROOT = "static"
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
@@ -129,4 +132,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 LOGIN_REDIRECT_URL = "home"
 LOGOUT_REDIRECT_URL = "home"
 
-ASGI_APPLICATION = "mumui.asgi.application"
+GRIP_URL = "http://pushpin:5561"
+
+CSRF_TRUSTED_ORIGINS = ["http://localhost:8080"]

nginx/Dockerfile

@@ -0,0 +1,7 @@
+FROM nginx:latest
+
+COPY nginx.conf /etc/nginx/nginx.conf
+
+EXPOSE 8080
+
+CMD ["nginx", "-g", "daemon off;"]

nginx/nginx.conf

@@ -0,0 +1,39 @@
+worker_processes 1;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    server {
+        listen 8080;
+
+        location /static/ {
+            alias /app/static/;
+        }
+
+        location / {
+            proxy_pass http://pushpin:7999;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        # location / {
+        #     proxy_pass http://mumui:8000;
+        #     proxy_set_header Host $host;
+        #     proxy_set_header X-Real-IP $remote_addr;
+        #     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        # }
+ 
+        # location /events {
+        #     proxy_pass http://pushpin:7999/stream;
+        #     proxy_set_header Host $host;
+        #     # proxy_set_header X-Real-IP $remote_addr;
+        #     # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        #     proxy_http_version 1.1;
+        #     # proxy_set_header Upgrade $http_upgrade;
+        #     proxy_set_header Connection "";
+        # }
+    }
+}

pushpin/Dockerfile

@@ -0,0 +1,3 @@
+FROM fanout/pushpin:latest
+
+RUN echo "* mumui:8000" > /etc/pushpin/routes

requirements-dev.txt

@@ -0,0 +1,2 @@
+black==23.9.1
+ruff==0.0.290

requirements.txt

@@ -1,43 +1,7 @@
-asgiref==3.7.2
-attrs==23.1.0
-autobahn==23.6.2
-Automat==22.10.0
-black==23.9.1
-certifi==2023.7.22
-cffi==1.15.1
-channels==3.0.5
-charset-normalizer==3.2.0
-click==8.1.7
-constantly==15.1.0
-cryptography==41.0.3
-daphne==3.0.2
 Django==4.2.5
+celery==5.3.4
 django-eventstream==4.5.1
-django-grip==3.4.0
+redis==4.6.0
-gripcontrol==4.2.0
+uwsgi==2.0.22
-hyperlink==21.0.0
+psycopg2-binary==2.9.7
-idna==3.4
+supervisor==4.2.5
-incremental==22.10.0
-MarkupSafe==2.1.3
-mypy-extensions==1.0.0
-packaging==23.1
-pathspec==0.11.2
-platformdirs==3.10.0
-pubcontrol==3.5.0
-pyasn1==0.5.0
-pyasn1-modules==0.3.0
-pycparser==2.21
-PyJWT==2.8.0
-pyOpenSSL==23.2.0
-requests==2.31.0
-ruff==0.0.290
-service-identity==23.1.0
-six==1.16.0
-sqlparse==0.4.4
-tomli==2.0.1
-Twisted==23.8.0
-txaio==23.1.1
-typing_extensions==4.7.1
-urllib3==2.0.4
-Werkzeug==2.3.7
-zope.interface==6.0

startup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+pip install -r requirements.txt
+
+python manage.py makemigrations
+python manage.py migrate
+
+DJANGO_SUPERUSER_PASSWORD=admin python manage.py createsuperuser --noinput --username admin --email admin@admin.fr
+
+supervisord -c /app/supervisord.conf

supervisord.conf

@@ -0,0 +1,23 @@
+[supervisord]
+nodaemon=true
+
+[program:mumui_uwsgi]
+command=/usr/local/bin/uwsgi --ini /app/uwsgi.ini --py-autoreload 2
+directory=/app
+user=nobody
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+
+# TODO(rmanach): add watchdog to restart celery on *.py files changes
+[program:mumui_celery]
+command=/usr/local/bin/celery -A mumui worker --loglevel=info
+directory=/app
+user=nobody
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0

templates/deployment/board.html

@@ -23,6 +23,7 @@
         <th>type</th>
         <th>status</th>
         <th></th>
+        <th></th>
       </tr>
       {% for deployment in page_obj %}
       <tr id="{{ deployment.id }}">
@@ -34,6 +35,14 @@
             <button>details</button>
           </a>
         </th>
+        {% if deployment.status == "FAILED" or deployment.status == "READY" %}
+        <th name="deploy">
+          <form action="{% url 'deployment-launch' deployment.id %}?page={{ page_obj.number }}" method="post">
+            {% csrf_token %}
+            <input type="submit" value="deploy">
+          </form>
+        </th>
+        {% endif %}
       </tr>
       {% endfor %}
     </table>
@@ -72,13 +81,35 @@
     console.log('connected');
   };
 
-  es.onerror = function () {
+  es.addEventListener('stream-error', function (e) {
+    es.close();
+    message = JSON.parse(e.data);
+    console.log('stream error: ' + message.condition + ': ' + message.text);
+  }, false);
+
+  es.onerror = function (e) {
     console.log('connection error');
   };
 
-  es.addEventListener('deployment', function (e) {
+  // listening on `message` events and update the corresponding table line.
-    e = JSON.parse(e.data);
+  // If the status is `FAILED`, we reload the window to ensure a new csrf_token.
-    console.log('stream reset: ' + JSON.stringify(e.channels));
+  es.addEventListener('message', function (e) {
+    message = JSON.parse(e.data);
+    console.log("id: " + message.id);
+    console.log("status: " + message.status);
+
+    const tr = document.getElementById(message.id);
+    if (tr) {
+      var th = tr.querySelector("th[name='status']");
+      th.innerHTML = message.status;
+
+      if (message.status == "FAILED") {
+        setTimeout(() => {
+          window.location.reload();
+        }, 2000);
+      }
+    }
+
   }, false);
 };
 </script>

uwsgi.ini

@@ -0,0 +1,7 @@
+[uwsgi]
+http = :8000
+module = mumui.wsgi:application
+master = True
+processes = 4
+threads = 2
+chdir = /app