# Copyright © The Debusine Developers
# See the AUTHORS file at the top-level directory of this distribution
#
# This file is part of Debusine. It is subject to the license terms
# in the LICENSE file found in the top-level directory of this
# distribution. No part of Debusine, including this file, may be copied,
# modified, propagated, or distributed except according to the terms
# contained in the LICENSE file.

"""debusine-admin to perform storage maintenance."""

from collections.abc import Callable
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, NoReturn

from django.core.management import CommandParser
from django.utils import timezone

from debusine.db.models import FileInStore, FileStore, FileUpload
from debusine.django.management.debusine_base_command import DebusineBaseCommand
from debusine.project import settings
from debusine.server.file_backend.interface import FileBackendEntryInterface


class Command(DebusineBaseCommand):
    """Command to report and fix inconsistencies."""

    help = "Report and fix inconsistencies in Debusine's DB and storage"

    def add_arguments(self, parser: CommandParser) -> None:
        """Add CLI arguments for the vacuum_storage command."""
        parser.add_argument(
            "--dry-run",
            help="Trial run: do not modify anything",
            action="store_true",
        )

    def handle(self, *args: Any, **options: Any) -> NoReturn:
        """Run the checks and fixes."""
        dry_run = options["dry_run"]
        self._delete_unreferenced_files(dry_run=dry_run)
        self._delete_empty_directories_in_the_store_directory(dry_run=dry_run)
        self._delete_unfinished_uploads(dry_run=dry_run)
        self._delete_orphan_files_from_upload_directory(dry_run=dry_run)
        missing_files_from_stores_correct = (
            self._report_missing_files_stores_referenced_by_the_db()
        )
        missing_files_from_upload_correct = (
            self._report_missing_files_referenced_by_db_file_upload()
        )

        raise SystemExit(
            int(
                not missing_files_from_stores_correct
                or not missing_files_from_upload_correct
            )
        )

    def _delete_unreferenced_files(self, *, dry_run: bool) -> None:
        """Delete files in stores that are not referenced by the DB."""
        self.print_verbose("Checking orphan files from stores")
        one_day_ago = timezone.now() - timedelta(days=1)

        # We consider only files older than a day old in order to allow a
        # grace period for files that are in the process of being created,
        # to avoid possible race conditions where this job deletes them
        # while we're still in the middle of creating references to them.
        # TODO: At the moment we can only filter by mtime in some backends,
        # although this isn't an inherent limitation.
        for store in FileStore.objects.filter(
            backend__in={
                FileStore.BackendChoices.LOCAL,
                FileStore.BackendChoices.S3,
            }
        ):
            established_entries_in_storage = set(
                store.get_backend_object().list_entries(
                    mtime_filter=lambda mtime: mtime < one_day_ago
                )
            )
            entries_in_db = self._entries_from_files_in_file_store_db(store)
            entries_to_delete = established_entries_in_storage - entries_in_db

            for entry_to_delete in entries_to_delete:
                if not dry_run:
                    entry_to_delete.remove()

                self.print_verbose(f"Deleted {entry_to_delete}")

    @staticmethod
    def _path_older_than_one_day(path: Path) -> bool:
        one_day_ago = datetime.now() - timedelta(days=1)
        file_modification_time = datetime.fromtimestamp(path.stat().st_mtime)
        return file_modification_time < one_day_ago

    def _delete_empty_directories_in_the_store_directory(
        self, *, dry_run: bool
    ) -> None:
        """
        Delete empty directories in the store directory.

        This method might leave new empty directories (by design).

        If the directory a/b/c is deleted: now the directory a/b might be empty.
        This method could then check if a/b is empty but:

        - mtime of a/b is the recent date
        - if it deleted a/b it could be that the LocalFileBackend is adding
           a file in a/b and could fail (since LocalFileBackend is creating
           the subdirectories incrementally).

        To fix this it could have a locking mechanism (DB based or file
        based).

        For now: empty directories might be eventually deleted but not
        "right now".
        """
        self.print_verbose("Checking empty directories")

        store_directory_directories = self._list_directory(
            settings.DEBUSINE_STORE_DIRECTORY,
            directories_only=True,
            filters=[self._path_older_than_one_day],
        )

        def path_length(directory: Path) -> int:
            return len(str(directory))

        store_directories_sorted = sorted(
            store_directory_directories, key=path_length, reverse=True
        )

        for directory in store_directories_sorted:
            # The directory is empty (and was modified more than a day ago):
            # it can be deleted
            if not dry_run:
                try:
                    directory.rmdir()
                except OSError:
                    # Probably a file has been added between _list_directory()
                    # call and the rmdir()
                    continue
            self.print_verbose(f"Deleted empty directory {directory}")

    def _delete_unfinished_uploads(self, *, dry_run: bool) -> None:
        """Delete from DB and disk uploads that started more than 1 day ago."""
        self.print_verbose("Checking unfinished file uploads")

        one_day_ago = timezone.now() - timedelta(days=1)

        old_file_uploads = FileUpload.objects.filter(
            last_activity_at__lt=one_day_ago
        )

        for file_upload in old_file_uploads:
            if not dry_run:
                file_upload.delete()
                Path(file_upload.path).unlink(missing_ok=True)

            self.print_verbose(f"Deleted unfinished upload {file_upload}")

    def _delete_orphan_files_from_upload_directory(
        self, *, dry_run: bool
    ) -> None:
        """Delete files from the upload directory that are not referenced."""
        self.print_verbose("Checking orphan files in upload directory")

        paths_in_disk = self._list_directory(
            settings.DEBUSINE_UPLOAD_DIRECTORY,
            filters=[self._path_older_than_one_day],
        )

        debusine_upload_directory = Path(settings.DEBUSINE_UPLOAD_DIRECTORY)

        paths_in_db = set()
        for path in FileUpload.objects.all().values_list("path", flat=True):
            paths_in_db.add(debusine_upload_directory / path)

        files_to_remove = paths_in_disk - paths_in_db

        for file_to_remove in files_to_remove:
            if not dry_run:
                file_to_remove.unlink(missing_ok=True)

            self.print_verbose(f"Deleted {file_to_remove}")

    def _report_missing_files_stores_referenced_by_the_db(self) -> bool:
        """
        Report DB referencing files that do not exist in storage.

        It cannot be fixed automatically.
        """
        self.print_verbose("Checking missing files from stores")
        ok = True

        for store in FileStore.objects.order_by("id"):
            # TODO: We should do this in a way that doesn't require holding
            # both sets in memory.
            entries_in_storage = set(store.get_backend_object().list_entries())
            entries_in_db = self._entries_from_files_in_file_store_db(store)
            orphan_entries = entries_in_db - entries_in_storage

            for orphan_entry in orphan_entries:
                self.stderr.write(
                    f"File in FileStore {store.name} but not in storage: "
                    f"{orphan_entry}"
                )

            ok = ok and (len(orphan_entries) == 0)

        return ok

    @staticmethod
    def _entries_from_files_in_file_store_db(
        store: FileStore,
    ) -> set[FileBackendEntryInterface[Any, Any]]:
        store_backend = store.get_backend_object()
        # TODO: This should probably also filter to files that are in any of
        # the scopes configured to use this store.  After `debusine-admin
        # scope manage --remove-file-store`, it's possible for the store to
        # contain files that are no longer in any of its scopes, and those
        # should be cleaned up.
        return {
            store_backend.get_entry(file_in_store.file)
            for file_in_store in FileInStore.objects.filter(store=store)
        }

    def _report_missing_files_referenced_by_db_file_upload(self) -> bool:
        """Report DB referencing upload files that do not exist in disk."""
        self.print_verbose("Checking missing files from upload")

        files_in_disk = self._list_directory(settings.DEBUSINE_UPLOAD_DIRECTORY)

        files_in_db = set()
        for file_in_db in (
            FileUpload.objects.all()
            .order_by("path")
            .values_list("path", flat=True)
        ):
            files_in_db.add(
                Path(settings.DEBUSINE_UPLOAD_DIRECTORY) / file_in_db
            )

        orphan_files = files_in_db - files_in_disk

        for orphan_file in orphan_files:
            self.stderr.write(
                f"File in FileUpload but not on disk: {orphan_file}"
            )

        return len(orphan_files) == 0

    @staticmethod
    def _list_directory(
        directory: Path | str,
        *,
        files_only: bool = False,
        directories_only: bool = False,
        filters: list[Callable[[Path], bool]] | None = None,
    ) -> set[Path]:
        if files_only and directories_only:
            raise ValueError(
                'Parameters "files_only" and "directories_only" '
                'are incompatible'
            )

        final_filters: list[Callable[[Path], bool]] = []

        if files_only:
            final_filters.append(lambda x: x.is_file())

        if directories_only:
            final_filters.append(lambda x: x.is_dir())

        if filters is not None:
            final_filters.extend(filters)

        # Pathlib.glob documentation: "Using the “**” pattern in large
        # directory trees may consume an inordinate amount of time."
        # Consider changing this with something else (e.g. os.walk; or
        # yielding paths and checking the DB per each file/set of files if
        # this becomes a problem).
        paths = Path(directory).glob("**/*")

        filtered_paths = set()

        for path in paths:
            if all(f(path) for f in final_filters):
                filtered_paths.add(path)

        return filtered_paths
