rmanach/mumui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dockerize app + add celery + impl pushpin for message events

Browse Source
This commit is contained in:
rmanach 2023-09-18 14:29:37 +02:00
parent bd11ddbd60
commit 5d51bc6637
28 changed files with 353 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -2,3 +2,6 @@
__pycache__
db.sqlite3
venv
static
*.log
*.pid

18
Dockerfile Normal file
View File

@ -0,0 +1,18 @@
FROM python:3.10
ENV PYTHONUNBUFFERED 1
ENV DJANGO_SETTINGS_MODULE mumui.settings
RUN mkdir /app
WORKDIR /app
RUN apt update && apt install -y
COPY startup.sh /app/
COPY requirements.txt /app/
RUN pip install -r requirements.txt
ENTRYPOINT [ "/app/startup.sh" ]
EXPOSE 8000

16
Makefile
View File

@ -1,4 +1,4 @@
#TODO(rmanach): add a pyproject.toml
# TODO(rmanach): add a pyproject.toml
format:
black deployment/*.py
@ -12,5 +12,17 @@ migrations:
python manage.py makemigrations
python manage.py migrate
django:
docker build . -t mumui:local
pushpin-local:
cd pushpin && docker build . -t pushpin:local
nginx-local:
cd nginx && docker build . -t nginx:local
run:
python manage.py runserver
docker compose up
stop:
docker compose down

36
deployment/migrations/0001_initial.py
View File

@ -1,9 +1,8 @@
# Generated by Django 4.2 on 2023-09-15 08:14
# Generated by Django 4.2.5 on 2023-09-18 08:14
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import uuid
class Migration(migrations.Migration):
@ -17,21 +16,32 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name="Deployment",
fields=[
(
"id",
models.UUIDField(
default=uuid.UUID("d7e9833f-e912-4640-8bf0-19e114a1136f"),
primary_key=True,
serialize=False,
),
),
("id", models.UUIDField(primary_key=True, serialize=False)),
("name", models.TextField()),
(
"type",
models.CharField(
choices=[("S", "Light"), ("M", "Medium"), ("L", "Heavy")],
default="S",
max_length=1,
choices=[
("SLIM", "Slim"),
("MEDIUM", "Medium"),
("LARGE", "Large"),
],
default="SLIM",
max_length=6,
),
),
(
"status",
models.CharField(
choices=[
("READY", "Ready"),
("PENDING", "Pending"),
("RUNNING", "Running"),
("SUCCESS", "Success"),
("FAILED", "Failed"),
],
default="READY",
max_length=7,
),
),
(

36
deployment/migrations/0002_deployment_status_alter_deployment_id.py
View File

@ -1,36 +0,0 @@
# Generated by Django 4.2 on 2023-09-15 08:31
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
("deployment", "0001_initial"),
]
operations = [
migrations.AddField(
model_name="deployment",
name="status",
field=models.CharField(
choices=[
("PENDING", "Pending"),
("RUNNING", "Running"),
("SUCCESS", "Success"),
("FAILED", "Failed"),
],
default="PENDING",
max_length=7,
),
),
migrations.AlterField(
model_name="deployment",
name="id",
field=models.UUIDField(
default=uuid.UUID("7909f7f2-0ee4-45f6-b8d4-beb5df8efc73"),
primary_key=True,
serialize=False,
),
),
]

24
deployment/migrations/0003_alter_deployment_id_alter_deployment_status.py
View File

@ -1,24 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-16 08:16
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
('deployment', '0002_deployment_status_alter_deployment_id'),
]
operations = [
migrations.AlterField(
model_name='deployment',
name='id',
field=models.UUIDField(default=uuid.UUID('8584555b-51ea-46a4-aebc-f7a8ab8670a4'), primary_key=True, serialize=False),
),
migrations.AlterField(
model_name='deployment',
name='status',
field=models.CharField(choices=[('READY', 'Ready'), ('PENDING', 'Pending'), ('RUNNING', 'Running'), ('SUCCESS', 'Success'), ('FAILED', 'Failed')], default='READY', max_length=7),
),
]

47
deployment/migrations/0004_alter_deployment_id_alter_deployment_status_and_more.py
View File

@ -1,47 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-16 08:43
import deployment.models
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
("deployment", "0003_alter_deployment_id_alter_deployment_status"),
]
operations = [
migrations.AlterField(
model_name="deployment",
name="id",
field=models.UUIDField(
default=uuid.UUID("d29e958f-7b3d-43a8-bc17-a40ab4184dc6"),
primary_key=True,
serialize=False,
),
),
migrations.AlterField(
model_name="deployment",
name="status",
field=models.CharField(
choices=[
("READY", "Ready"),
("PENDING", "Pending"),
("RUNNING", "Running"),
("SUCCESS", "Success"),
("FAILED", "Failed"),
],
default=deployment.models.Status["READY"],
max_length=7,
),
),
migrations.AlterField(
model_name="deployment",
name="type",
field=models.CharField(
choices=[("SLIM", "Slim"), ("MEDIUM", "Medium"), ("LARGE", "Large")],
default=deployment.models.Type["SLIM"],
max_length=6,
),
),
]

22
deployment/migrations/0005_alter_deployment_id.py
View File

@ -1,22 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-16 08:43
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
("deployment", "0004_alter_deployment_id_alter_deployment_status_and_more"),
]
operations = [
migrations.AlterField(
model_name="deployment",
name="id",
field=models.UUIDField(
default=uuid.UUID("ec7ebcb5-420a-47c3-ba07-e8c340895865"),
primary_key=True,
serialize=False,
),
),
]

46
deployment/migrations/0006_alter_deployment_id_alter_deployment_status_and_more.py
View File

@ -1,46 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-16 08:45
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
("deployment", "0005_alter_deployment_id"),
]
operations = [
migrations.AlterField(
model_name="deployment",
name="id",
field=models.UUIDField(
default=uuid.UUID("b8feb7aa-9c79-478b-ba91-df2e03f4145b"),
primary_key=True,
serialize=False,
),
),
migrations.AlterField(
model_name="deployment",
name="status",
field=models.CharField(
choices=[
("READY", "Ready"),
("PENDING", "Pending"),
("RUNNING", "Running"),
("SUCCESS", "Success"),
("FAILED", "Failed"),
],
default="Ready",
max_length=7,
),
),
migrations.AlterField(
model_name="deployment",
name="type",
field=models.CharField(
choices=[("SLIM", "Slim"), ("MEDIUM", "Medium"), ("LARGE", "Large")],
default="Slim",
max_length=6,
),
),
]

22
deployment/migrations/0007_alter_deployment_id.py
View File

@ -1,22 +0,0 @@
# Generated by Django 4.2.5 on 2023-09-16 17:21
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
("deployment", "0006_alter_deployment_id_alter_deployment_status_and_more"),
]
operations = [
migrations.AlterField(
model_name="deployment",
name="id",
field=models.UUIDField(
default=uuid.UUID("0cc4ae03-f52e-4f3a-9fe1-fecda707b20e"),
primary_key=True,
serialize=False,
),
),
]

15
deployment/models.py
View File

@ -4,9 +4,6 @@ from uuid import uuid4
from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import pre_save, pre_delete
from django.dispatch import receiver
from django_eventstream import send_event
class Type(Enum):
@ -32,21 +29,15 @@ class Status(Enum):
class Deployment(models.Model):
id = models.UUIDField(primary_key=True, default=uuid4())
id = models.UUIDField(primary_key=True)
name = models.TextField(null=False, blank=False)
type = models.CharField(
max_length=6, choices=Type.into_choices(), default=Type.SLIM.value
max_length=6, choices=Type.into_choices(), default=Type.SLIM.name
)
status = models.CharField(
max_length=7, choices=Status.into_choices(), default=Status.READY.value
max_length=7, choices=Status.into_choices(), default=Status.READY.name
)
user = models.ForeignKey(User, on_delete=models.CASCADE)
def __str__(self):
return f"{self.name} | {self.type} | {self.status}"
@receiver(pre_save, sender=Deployment)
@receiver(pre_delete, sender=Deployment)
def deployment_update_handler(sender, instance, **kwargs):
send_event("deployment", "update", {"id": instance.id, "status": instance.status})

32
deployment/tasks.py Normal file
View File

@ -0,0 +1,32 @@
import time
import random
from uuid import UUID
from celery import shared_task
from django_eventstream import send_event
from deployment.models import Deployment, Type, Status
@shared_task
def deploy(deployment_id: UUID):
deploy = Deployment.objects.get(id=deployment_id)
deploy.status = Status.RUNNING.name
deploy.save()
match deploy.type:
case Type.SLIM.name:
time.sleep(10)
case Type.MEDIUM.name:
time.sleep(60)
case Type.LARGE.name:
time.sleep(120)
deploy.status = (
Status.FAILED.name if random.randint(0, 10) % 2 != 0 else Status.SUCCESS.name
)
deploy.save()
send_event("test", "message", {"id": deploy.id, "status": deploy.status})
print("event sent")

8
deployment/urls.py
View File

@ -1,8 +1,12 @@
from django.urls import path
from deployment.views import index, create, details
import django_eventstream
from django.urls import path, include
from deployment.views import index, create, details, deploy
urlpatterns = [
path("", index, name="deployment"),
path("create", create, name="deployment-create"),
path("<uuid:deployment_id>", details, name="deployment-details"),
path("<uuid:deployment_id>/deploy", deploy, name="deployment-launch"),
path("events/", include(django_eventstream.urls), {"channels": ["test"]}),
]

17
deployment/views.py
View File

@ -10,6 +10,7 @@ from django.shortcuts import render, get_object_or_404
from deployment.forms import DeploymentForm
from deployment.models import Deployment, Status
from deployment.tasks import deploy as launch_deploy
def index(request):
@ -20,10 +21,24 @@ def index(request):
page_obj = paginator.get_page(page_number)
return render(
request, "deployment/board.html", {"page_obj": page_obj, "url": "/events/"}
request, "deployment/board.html", {"page_obj": page_obj, "url": "events/"}
)
def deploy(request, deployment_id):
deployment = get_object_or_404(Deployment, id=deployment_id)
if request.method == "POST":
deployment.status = Status.PENDING.name
deployment.save()
launch_deploy.delay(deployment_id)
if page := request.GET.get("page", ""):
return HttpResponseRedirect(f"/deployment?page={page}")
return HttpResponseRedirect("/deployment")
def details(request, deployment_id):
deployment = get_object_or_404(Deployment, id=deployment_id)

60
docker-compose.yml Normal file
View File

@ -0,0 +1,60 @@
version: '3'
services:
redis:
image: redis/redis-stack-server:latest
container_name: redis
networks:
- mumui_network
volumes:
- redis_data:/data
pushpin:
image: pushpin:local
container_name: pushpin
networks:
- mumui_network
depends_on:
- mumui
postgres:
image: postgres:latest
container_name: postgres
environment:
POSTGRES_DB: mumui
POSTGRES_USER: test
POSTGRES_PASSWORD: test
networks:
- mumui_network
volumes:
- postgres_data:/var/lib/postgresql/data
mumui:
image: mumui:local
container_name: mumui
networks:
- mumui_network
volumes:
- .:/app
depends_on:
- postgres
nginx:
image: nginx:local
container_name: nginx
networks:
- mumui_network
volumes:
- ./static:/app/static
ports:
- "8080:8080"
depends_on:
- mumui
networks:
mumui_network:
driver: bridge
volumes:
redis_data:
postgres_data:

3
mumui/__init__.py
View File

@ -0,0 +1,3 @@
from mumui.celery import app as celery_app
__all__ = ("celery_app",)

46
mumui/asgi.py
View File

@ -7,28 +7,28 @@ For more information on this file, see
https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
from django.urls import path, re_path
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.auth import AuthMiddlewareStack
import django_eventstream
# import os
# from django.core.asgi import get_asgi_application
# from django.urls import path, re_path
# from channels.routing import ProtocolTypeRouter, URLRouter
# from channels.auth import AuthMiddlewareStack
# import django_eventstream
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings")
# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings")
application = ProtocolTypeRouter(
{
"http": URLRouter(
[
path(
"events/",
AuthMiddlewareStack(
URLRouter(django_eventstream.routing.urlpatterns)
),
{"channels": ["test"]},
),
re_path(r"", get_asgi_application()),
]
),
}
)
# application = ProtocolTypeRouter(
# {
# "http": URLRouter(
# [
# path(
# "events/",
# AuthMiddlewareStack(
# URLRouter(django_eventstream.routing.urlpatterns)
# ),
# {"channels": ["deployment"]},
# ),
# re_path(r"", get_asgi_application()),
# ]
# ),
# }
# )

10
mumui/celery.py Normal file
View File

@ -0,0 +1,10 @@
import os
from celery import Celery
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mumui.settings")
app = Celery("mumui", broker="redis://redis:6379/0")
app.config_from_object("django.conf:settings", namespace="CELERY")
app.autodiscover_tasks()

17
mumui/settings.py
View File

@ -25,7 +25,7 @@ SECRET_KEY = "django-insecure-_c56%%c8%g%@5(3&thxi7ku2a&wst8lik*8@l0=#)ar)s86g36
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
ALLOWED_HOSTS = ["*"]
# Application definition
@ -38,7 +38,6 @@ INSTALLED_APPS = [
"django.contrib.messages",
"django.contrib.staticfiles",
"deployment",
"channels",
"django_eventstream",
]
@ -79,12 +78,15 @@ WSGI_APPLICATION = "mumui.wsgi.application"
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": BASE_DIR / "db.sqlite3",
"ENGINE": "django.db.backends.postgresql",
"NAME": "mumui",
"USER": "test",
"PASSWORD": "test",
"HOST": "postgres",
"PORT": "5432",
}
}
# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
@ -120,6 +122,7 @@ USE_TZ = True
# https://docs.djangoproject.com/en/4.2/howto/static-files/
STATIC_URL = "static/"
STATIC_ROOT = "static"
# Default primary key field type
# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
@ -129,4 +132,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
LOGIN_REDIRECT_URL = "home"
LOGOUT_REDIRECT_URL = "home"
ASGI_APPLICATION = "mumui.asgi.application"
GRIP_URL = "http://pushpin:5561"
CSRF_TRUSTED_ORIGINS = ["http://localhost:8080"]

7
nginx/Dockerfile Normal file
View File

@ -0,0 +1,7 @@
FROM nginx:latest
COPY nginx.conf /etc/nginx/nginx.conf
EXPOSE 8080
CMD ["nginx", "-g", "daemon off;"]

39
nginx/nginx.conf Normal file
View File

@ -0,0 +1,39 @@
worker_processes 1;
events {
worker_connections 1024;
}
http {
server {
listen 8080;
location /static/ {
alias /app/static/;
}
location / {
proxy_pass http://pushpin:7999;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# location / {
# proxy_pass http://mumui:8000;
# proxy_set_header Host $host;
# proxy_set_header X-Real-IP $remote_addr;
# proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# }
# location /events {
# proxy_pass http://pushpin:7999/stream;
# proxy_set_header Host $host;
# # proxy_set_header X-Real-IP $remote_addr;
# # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# proxy_http_version 1.1;
# # proxy_set_header Upgrade $http_upgrade;
# proxy_set_header Connection "";
# }
}
}

3
pushpin/Dockerfile Normal file
View File

@ -0,0 +1,3 @@
FROM fanout/pushpin:latest
RUN echo "* mumui:8000" > /etc/pushpin/routes

2
requirements-dev.txt Normal file
View File

@ -0,0 +1,2 @@
black==23.9.1
ruff==0.0.290

46
requirements.txt
View File

@ -1,43 +1,7 @@
asgiref==3.7.2
attrs==23.1.0
autobahn==23.6.2
Automat==22.10.0
black==23.9.1
certifi==2023.7.22
cffi==1.15.1
channels==3.0.5
charset-normalizer==3.2.0
click==8.1.7
constantly==15.1.0
cryptography==41.0.3
daphne==3.0.2
Django==4.2.5
celery==5.3.4
django-eventstream==4.5.1
django-grip==3.4.0
gripcontrol==4.2.0
hyperlink==21.0.0
idna==3.4
incremental==22.10.0
MarkupSafe==2.1.3
mypy-extensions==1.0.0
packaging==23.1
pathspec==0.11.2
platformdirs==3.10.0
pubcontrol==3.5.0
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycparser==2.21
PyJWT==2.8.0
pyOpenSSL==23.2.0
requests==2.31.0
ruff==0.0.290
service-identity==23.1.0
six==1.16.0
sqlparse==0.4.4
tomli==2.0.1
Twisted==23.8.0
txaio==23.1.1
typing_extensions==4.7.1
urllib3==2.0.4
Werkzeug==2.3.7
zope.interface==6.0
redis==4.6.0
uwsgi==2.0.22
psycopg2-binary==2.9.7
supervisor==4.2.5

9
startup.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
pip install -r requirements.txt
python manage.py makemigrations
python manage.py migrate
DJANGO_SUPERUSER_PASSWORD=admin python manage.py createsuperuser --noinput --username admin --email admin@admin.fr
supervisord -c /app/supervisord.conf

23
supervisord.conf Normal file
View File

@ -0,0 +1,23 @@
[supervisord]
nodaemon=true
[program:mumui_uwsgi]
command=/usr/local/bin/uwsgi --ini /app/uwsgi.ini --py-autoreload 2
directory=/app
user=nobody
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
# TODO(rmanach): add watchdog to restart celery on *.py files changes
[program:mumui_celery]
command=/usr/local/bin/celery -A mumui worker --loglevel=info
directory=/app
user=nobody
autostart=true
autorestart=true
redirect_stderr=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0

39
templates/deployment/board.html
View File

@ -23,6 +23,7 @@
<th>type</th>
<th>status</th>
<th></th>
<th></th>
</tr>
{% for deployment in page_obj %}
<tr id="{{ deployment.id }}">
@ -34,6 +35,14 @@
<button>details</button>
</a>
</th>
{% if deployment.status == "FAILED" or deployment.status == "READY" %}
<th name="deploy">
<form action="{% url 'deployment-launch' deployment.id %}?page={{ page_obj.number }}" method="post">
{% csrf_token %}
<input type="submit" value="deploy">
</form>
</th>
{% endif %}
</tr>
{% endfor %}
</table>
@ -72,13 +81,35 @@
console.log('connected');
};
es.onerror = function () {
es.addEventListener('stream-error', function (e) {
es.close();
message = JSON.parse(e.data);
console.log('stream error: ' + message.condition + ': ' + message.text);
}, false);
es.onerror = function (e) {
console.log('connection error');
};
es.addEventListener('deployment', function (e) {
e = JSON.parse(e.data);
console.log('stream reset: ' + JSON.stringify(e.channels));
// listening on `message` events and update the corresponding table line.
// If the status is `FAILED`, we reload the window to ensure a new csrf_token.
es.addEventListener('message', function (e) {
message = JSON.parse(e.data);
console.log("id: " + message.id);
console.log("status: " + message.status);
const tr = document.getElementById(message.id);
if (tr) {
var th = tr.querySelector("th[name='status']");
th.innerHTML = message.status;
if (message.status == "FAILED") {
setTimeout(() => {
window.location.reload();
}, 2000);
}
}
}, false);
};
</script>

7
uwsgi.ini Normal file
View File

@ -0,0 +1,7 @@
[uwsgi]
http = :8000
module = mumui.wsgi:application
master = True
processes = 4
threads = 2
chdir = /app