Почему в Django не передаёт Celery task?
Запускаю в git codespace свой docker образ. Не понимаю, почему celery не выполняет ни один task.
В логах контейнера web вижу
nnpf-web-1 | [27/Mar/2023 21:54:44] "GET / HTTP/1.1" 200 4320
nnpf-web-1 | [27/Mar/2023 21:54:49] "POST / HTTP/1.1" 200 5990
nnpf-web-1 | [27/Mar/2023 21:54:50] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:51] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:51] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:52] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:54] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:55] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:56] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:57] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:58] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:54:59] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
nnpf-web-1 | [27/Mar/2023 21:55:00] "GET /task/8268acea-597d-4be2-b733-89d66d17777e/ HTTP/1.1" 200 77
логи celery
You have 18 unapplied migration(s). Your project may not work properly until you apply the migrations for app(s): admin, auth, contenttypes, sessions.
Run 'python manage.py migrate' to apply them.
March 27, 2023 - 21:58:20
Django version 4.0.6, using settings 'image_pnev.settings'
Starting development server at http://0.0.0.0:8000/
Quit the server with CONTROL-C.
логи redis
1:C 27 Mar 2023 21:57:53.145 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
1:C 27 Mar 2023 21:57:53.145 # Redis version=5.0.14, bits=64, commit=00000000, modified=0, pid=1, just started
1:C 27 Mar 2023 21:57:53.145 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
1:M 27 Mar 2023 21:57:53.146 * Running mode=standalone, port=6379.
1:M 27 Mar 2023 21:57:53.146 # Server initialized
1:M 27 Mar 2023 21:57:53.146 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
1:M 27 Mar 2023 21:57:53.146 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled.
1:M 27 Mar 2023 21:57:53.147 * DB loaded from disk: 0.000 seconds
1:M 27 Mar 2023 21:57:53.147 * Ready to accept connections
мой settings.py
"""
Django settings for image_pnev project.
Generated by 'django-admin startproject' using Django 4.0.6.
For more information on this file, see
https://docs.djangoproject.com/en/4.0/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.0/ref/settings/
"""
from pathlib import Path
import os
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '123'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'thumbnailer.apps.ThumbnailerConfig',
'widget_tweaks',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'image_pnev.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'image_pnev.wsgi.application'
# Database
# https://docs.djangoproject.com/en/4.0/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
CSRF_TRUSTED_ORIGINS = ['https://egorrybko-urban-engine-q5w57pp6jxpf4x5w-1337.preview.app.github.dev','https://127.0.0.1']
# Password validation
# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.0/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.0/howto/static-files/
STATIC_URL = 'static/'
# Default primary key field type
# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
# celery
#CELERY_BROKER_URL = 'redis://redis:6379/0'
#CELERY_RESULT_BACKEND = "redis"
#CELERY_RESULT_BACKEND = 'redis://redis:6379/0'
CELERY_BROKER_URL = os.environ.get("CELERY_BROKER", "redis://redis:6379/0")
CELERY_RESULT_BACKEND = os.environ.get("CELERY_BROKER", "redis://redis:6379/0")
#redis://localhost:6379
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = '/static/'
MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.abspath(os.path.join(BASE_DIR, 'media'))
IMAGES_DIR = os.path.join(MEDIA_ROOT, 'images')
if not os.path.exists(MEDIA_ROOT) or not os.path.exists(IMAGES_DIR):
os.makedirs(IMAGES_DIR)
мой viev.py
# thumbnailer/views.py
import os
from celery import current_app
from django import forms
from django.conf import settings
from django.http import JsonResponse
from django.shortcuts import render
from django.views import View
from .tasks import make_thumbnails
class FileUploadForm(forms.Form):
image_file = forms.ImageField(required=True)
class HomeView(View):
def get(self, request):
form = FileUploadForm()
return render(request, 'thumbnailer/home.html', { 'form': form })
def post(self, request):
form = FileUploadForm(request.POST, request.FILES)
context = {}
if form.is_valid():
file_path = os.path.join(settings.IMAGES_DIR, request.FILES['image_file'].name)
with open(file_path, 'wb+') as fp:
for chunk in request.FILES['image_file']:
fp.write(chunk)
task = make_thumbnails.delay(file_path, thumbnails=[(128, 128)])
context['task_id'] = task.id
context['task_status'] = task.status
return render(request, 'thumbnailer/home.html', context)
context['form'] = form
return render(request, 'thumbnailer/home.html', context)
class TaskView(View):
def get(self, request, task_id):
task = current_app.AsyncResult(task_id)
response_data = {'task_status': task.status, 'task_id': task.id}
if task.status == 'SUCCESS':
response_data['results'] = task.get()
return JsonResponse(response_data)
def main(reqest):
return render(reqest, 'thumbnailer/main.html')
def about_prj(reqest):
return render(reqest, 'thumbnailer/about_prj.html')
def about_us(reqest):
return render(reqest, 'thumbnailer/about_us.html')
мой task.py
# image_parroter/thumbnailer/tasks.py
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
# Any results you write to the current directory are saved as output.
import matplotlib.pyplot as plt
import seaborn as sns
import keras
from keras.models import Sequential
from keras.layers import Dense, Conv2D , MaxPool2D , Flatten , Dropout , BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report,confusion_matrix
from keras.callbacks import ReduceLROnPlateau
import cv2
import os
import pickle
import os
from zipfile import ZipFile
from celery import shared_task
from PIL import Image
from django.conf import settings
from pathlib import Path
@shared_task
def make_thumbnails(file_path, thumbnails=[]):
path, file = os.path.split(file_path)
file_name, ext = os.path.splitext(file)
zip_file = f"{file_name}.zip"
results = {'archive_path': f"{settings.MEDIA_URL}images/{zip_file}"}
try:
zipper = ZipFile(path+'/'+zip_file, 'w')
# labels = ['PNEUMONIA', 'NORMAL']
img_size = 150
x1 = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)
x2 = cv2.resize(x1, (img_size, img_size)) # Reshaping images to preferred size
x3 = np.array(x2) / 255
x3 = x3.reshape(-1, img_size, img_size, 1)
#history = pickle.load(open('trainHistoryDict', "rb"))
ROOT_DIR = Path(os.path.dirname(os.path.abspath(__file__))).resolve().parents[1].__str__()
model = keras.models.load_model(ROOT_DIR+'\image_pnev\model_01')
model.load_weights(ROOT_DIR+'\image_pnev\model_weights_01')
predictions = model.predict(x3)
predictions = predictions.reshape(1, -1)[0]
fig = plt.figure(figsize=(10, 10))
plt.imshow(x1, cmap='gray')
plt.title("Predicted Class {}".format(predictions[0])+'1 - normal, 0 - bad')
fig.savefig(file_path)
os.chdir(settings.IMAGES_DIR)
zipper.write(file)
os.remove(file_path)
zipper.close()
except IOError as e:
print(e)
return results
При запуске проекта локально, без docker логи web такие
[28/Mar/2023 10:56:59] "POST / HTTP/1.1" 200 5937
[28/Mar/2023 10:57:00] "GET /task/6c64add3-17ce-42ee-a406-4a0e35e7a909/ HTTP/1.1" 200 77
[28/Mar/2023 10:57:01] "GET /task/6c64add3-17ce-42ee-a406-4a0e35e7a909/ HTTP/1.1" 200 77
[28/Mar/2023 10:57:02] "GET /task/6c64add3-17ce-42ee-a406-4a0e35e7a909/ HTTP/1.1" 200 77
[28/Mar/2023 10:57:02] "GET /task/6c64add3-17ce-42ee-a406-4a0e35e7a909/ HTTP/1.1" 200 140
[28/Mar/2023 10:57:03] "GET /media/images/IM-0023-0001.zip HTTP/1.1" 200 62645
а celery видит task
[2023-03-28 10:56:29,716: INFO/MainProcess] celery@5CD11362MQ ready.
[2023-03-28 10:56:59,717: INFO/MainProcess] Task thumbnailer.tasks.make_thumbnails[6c64add3-17ce-42ee-a406-4a0e35e7a909] received
Источник: Stack Overflow на русском