python - 模块未找到错误: No module named 'social.models' when running celery worker

Question

我有以下项目结构:

SocialRating
- accounts
-- __init__.py
-- models.py
-- tasks.py

- instagram
-- __init__.py
-- crawler.py

- social
-- __init__.py
-- models.py # Not Django models

- celery_worker.sh

accounts/models.py 包含类:

from instagram.crawler import InstagramCrawler

class SocialProfile(models.Model):
    ...

    def collect_profile_stats(self):
        crawler = InstagramCrawler()
        record_data = crawler.get_profile_stats(self.account_id)

InstagramCrawler 类位于 instagram/crawler.py:

from social.models import SocialUserData, SocialCommentData, SocialProfileStatsRecordData


class InstagramCrawler(object):
    ...

并且social/models.py包含简单的数据类:

@dataclass
class SocialUserData(object):
    user_id: str


@dataclass
class SocialCommentData(object):
    social_id: str
    user: SocialUserData
    text: str


@dataclass
class SocialPostStatsRecordData(object):
    post_id: str
    subscribers_count: int
    likes_count: int
    reposts_count: int
    comments_count: int
    post_date: str


@dataclass
class SocialProfileStatsRecordData(object):
    subscribers_count: int

accounts/tasks.py:

@app.task(bind=True)
def update_profile_stats(self, social_profile_pk: int):
    social_profile = SocialProfile.objects.get(pk=social_profile_pk)

    logger.info(f'Updating profile stats for {social_profile}')

    social_profile.collect_profile_stats()

celery_worker.sh:

#!/usr/bin/env bash

celery -A SocialRating worker -l info

因此，当我运行 celery_worker.sh (从项目的根目录)时，它会引发:

Signal handler <bound method DjangoFixup.on_import_modules of <celery.fixups.django.DjangoFixup object at 0x106819d30>> raised: ModuleNotFoundError("No module named 'social.models'")
Traceback (most recent call last):
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/utils/dispatch/signal.py", line 288, in send
    response = receiver(signal=self, sender=sender, **named)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/fixups/django.py", line 82, in on_import_modules
    self.worker_fixup.validate_models()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/fixups/django.py", line 120, in validate_models
    self.django_setup()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/fixups/django.py", line 116, in django_setup
    django.setup()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/django/__init__.py", line 24, in setup
    apps.populate(settings.INSTALLED_APPS)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/django/apps/registry.py", line 112, in populate
    app_config.import_models()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/django/apps/config.py", line 198, in import_models
    self.models_module = import_module(models_module_name)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 728, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/Users/user/src/python/web/SocialRating/accounts/models.py", line 4, in <module>
    from instagram.crawler import InstagramCrawler
  File "/Users/user/src/python/web/SocialRating/instagram/crawler.py", line 7, in <module>
    from social.models import SocialUserData, SocialCommentData, SocialProfileStatsRecordData
ModuleNotFoundError: No module named 'social.models'
Traceback (most recent call last):
  File "/Users/user/src/python/web/SocialRating/.venv/bin/celery", line 10, in <module>
    sys.exit(main())
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/__main__.py", line 16, in main
    _main()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/celery.py", line 322, in main
    cmd.execute_from_commandline(argv)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/celery.py", line 496, in execute_from_commandline
    super(CeleryCommand, self).execute_from_commandline(argv)))
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/base.py", line 275, in execute_from_commandline
    return self.handle_argv(self.prog_name, argv[1:])
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/celery.py", line 488, in handle_argv
    return self.execute(command, argv)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/celery.py", line 420, in execute
    ).run_from_argv(self.prog_name, argv[1:], command=argv[0])
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/worker.py", line 223, in run_from_argv
    return self(*args, **options)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/base.py", line 238, in __call__
    ret = self.run(*args, **kwargs)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/bin/worker.py", line 257, in run
    **kwargs)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/worker/worker.py", line 96, in __init__
    self.app.loader.init_worker()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/loaders/base.py", line 114, in init_worker
    self.import_default_modules()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/loaders/base.py", line 108, in import_default_modules
    raise response
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/utils/dispatch/signal.py", line 288, in send
    response = receiver(signal=self, sender=sender, **named)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/fixups/django.py", line 82, in on_import_modules
    self.worker_fixup.validate_models()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/fixups/django.py", line 120, in validate_models
    self.django_setup()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/celery/fixups/django.py", line 116, in django_setup
    django.setup()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/django/__init__.py", line 24, in setup
    apps.populate(settings.INSTALLED_APPS)
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/django/apps/registry.py", line 112, in populate
    app_config.import_models()
  File "/Users/user/src/python/web/SocialRating/.venv/lib/python3.7/site-packages/django/apps/config.py", line 198, in import_models
    self.models_module = import_module(models_module_name)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 728, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/Users/user/src/python/web/SocialRating/accounts/models.py", line 4, in <module>
    from instagram.crawler import InstagramCrawler
  File "/Users/user/src/python/web/SocialRating/instagram/crawler.py", line 7, in <module>
    from social.models import SocialUserData, SocialCommentData, SocialProfileStatsRecordData
ModuleNotFoundError: No module named 'social.models'

但是当我直接从 django 命令使用 InstagramCrawler 时，它工作正常:

class Command(BaseCommand):

    def handle(self, *args, **options):
        self.test_account_followers_count()


    def test_account_followers_count(self):
        crawler = InstagramCrawler()
        print(crawler.get_subscribers_count('someacc'))

为什么？

Answer 1

当从 celery_worker.sh 运行 Celery 时，似乎没有加载 social.models 模块。

在运行celery_worker.sh之前尝试显式设置PYTHONPATH。例如:

export PYTHONPATH=.; ./celery_worker.sh