#!/usr/bin/python3
# encoding=utf-8
#
# Copyright © 2014-2016 Simon McVittie <smcv@debian.org>
# Copyright © 2015-2016 Alexandre Detiste <alexandre@detiste.be>
# SPDX-License-Identifier: GPL-2.0-or-later

from collections import defaultdict
from enum import Enum
import logging
import os
import shutil
import stat
import subprocess
import tempfile
import zipfile

import yaml

from .data import (HashedFile)
from .download import (Downloader, NotDownloadable, OutOfSpace)
from .gog import GOG
from .packaging import (PerPackageState, get_native_packaging_system)
from .paths import (DATADIR)
from .unpack import (TarUnpacker, ZipUnpacker)
from .unpack.innoextract import (InnoSetup)
from .unpack.umod import (Umod)
from .util import (TemporaryUmask,
        check_call,
        check_output,
        copy_with_substitutions,
        lang_score,
        mkdir_p,
        rm_rf,
        recursive_utime,
        which)

logging.basicConfig()
logger = logging.getLogger(__name__)

class FillResult(Enum):
    UNDETERMINED = 0
    IMPOSSIBLE = 1
    DOWNLOAD_NEEDED = 2
    COMPLETE = 3
    UPGRADE_NEEDED = 4
    DEACTIVATED = 5

    @property
    def is_possible(self):
        return self in (
            FillResult.COMPLETE, FillResult.DOWNLOAD_NEEDED
        )

    def __and__(self, other):
        if other is FillResult.UNDETERMINED:
            return self

        if self is FillResult.UNDETERMINED:
            return other

        if other is FillResult.IMPOSSIBLE or self is FillResult.IMPOSSIBLE:
            return FillResult.IMPOSSIBLE

        if other is FillResult.DEACTIVATED or self is FillResult.DEACTIVATED:
            return FillResult.DEACTIVATED

        if other is FillResult.UPGRADE_NEEDED or self is FillResult.UPGRADE_NEEDED:
            return FillResult.UPGRADE_NEEDED

        if other is FillResult.DOWNLOAD_NEEDED or self is FillResult.DOWNLOAD_NEEDED:
            return FillResult.DOWNLOAD_NEEDED

        return FillResult.COMPLETE

    def __or__(self, other):
        if other is FillResult.UNDETERMINED:
            return self

        if self is FillResult.UNDETERMINED:
            return other

        if other is FillResult.COMPLETE or self is FillResult.COMPLETE:
            return FillResult.COMPLETE

        if other is FillResult.DOWNLOAD_NEEDED or self is FillResult.DOWNLOAD_NEEDED:
            return FillResult.DOWNLOAD_NEEDED

        if other is FillResult.UPGRADE_NEEDED or self is FillResult.UPGRADE_NEEDED:
            return FillResult.UPGRADE_NEEDED

        if other is FillResult.DEACTIVATED or self is FillResult.DEACTIVATED:
            return FillResult.DEACTIVATED

        return FillResult.IMPOSSIBLE

class BinaryExecutablesNotAllowed(Exception):
    pass

class NoPackagesPossible(Exception):
    pass

class DownloadsFailed(Exception):
    pass

class DownloadNotAllowed(Exception):
    pass

class CDRipFailed(Exception):
    pass

def iter_fat_mounts(folder):
    with open('/proc/mounts', 'r', encoding='utf8') as mounts:
        for line in mounts.readlines():
            mount, vfstype = line.split(' ')[1:3]
            if vfstype in ('fat', 'vfat', 'ntfs'):
                path = os.path.join(mount, 'Program Files (x86)', folder)
                if os.path.isdir(path):
                    yield path
                path = os.path.join(mount, 'Program Files', folder)
                if os.path.isdir(path):
                    yield path
                path = os.path.join(mount, folder)
                if os.path.isdir(path):
                    yield path

class PackagingTask(object):
    def __init__(self, game, packaging=None, builder_packaging=None):
        # A GameData object.
        self.game = game

        # The packaging system for which we are generating packages
        self.__packaging = packaging

        # The packaging system used to find tools such as unrar
        self.__builder_packaging = builder_packaging

        # A temporary directory.
        self.__workdir = None

        # Clean up these directories on exit.
        self._cleanup_dirs = set()

        # Map from WantedFile name to whether we can get it.
        # file_status[x] is COMPLETE if and only if either
        # found[x] exists, or x has alternative y and found[y] exists.
        self.file_status = defaultdict(lambda: FillResult.UNDETERMINED)

        # Map from WantedFile name to the absolute or relative path of
        # a matching file on disk.
        # { 'baseq3/pak1.pk3': '/usr/share/games/quake3/baseq3/pak1.pk3' }
        self.found = {}

        # Map from Package name to whether we can do it
        self.package_status = defaultdict(lambda: FillResult.UNDETERMINED)

        # Set of executables we wanted but don't have
        self.missing_tools = set()

        # Set of filenames we couldn't unpack, or already unpacked
        self.unpack_tried = set()

        # Block device from which to rip audio
        self.cd_device = None

        # Found CD tracks
        # e.g. { 'quake-music': { 'id1/music/track02.ogg': '/usr/.../id1/music/track02.ogg' } }
        self.cd_tracks = {}

        # If true, be more verbose
        self.verbose = False

        # None or an existing directory in which to save downloaded files.
        self.save_downloads = None

        # Factory for a progress report (or None).
        self.progress_factory = lambda info=None: None

        self.downloader = None
        self.game.load_file_data()

        # Map from package that is not Architecture: all to dpkg architecture
        # e.g. { 'ut99-bin': 'amd64', 'quake4-bin': 'i386' }
        self.package_architectures = {}

    def __del__(self):
        self.__exit__(None, None, None)

    def __enter__(self):
        return self

    def __exit__(self, _et, _ev, _tb):
        for d in self._cleanup_dirs:
            shutil.rmtree(d, onerror=lambda func, path, ei:
                logger.warning('error removing "%s":' % path, exc_info=ei))
        self._cleanup_dirs = set()

    @property
    def packaging(self):
        """The PackagingSystem in use."""
        if self.__packaging is None:
            self.__packaging = get_native_packaging_system()

        return self.__packaging

    @property
    def builder_packaging(self):
        """The PackagingSystem on the system doing the build."""
        if self.__builder_packaging is None:
            self.__builder_packaging = get_native_packaging_system()

        return self.__builder_packaging

    def get_workdir(self):
        if self.__workdir is None:
            self.__workdir = tempfile.mkdtemp(prefix='gdptmp.')
            self._cleanup_dirs.add(self.__workdir)
        return self.__workdir

    def use_file(self, found, candidates, path, hashes=None):
        logger.debug('found %s at %s', found, path)
        size = os.stat(path).st_size

        assert candidates

        remaining = set()

        for wanted in candidates:
            if wanted.size is None or wanted.size == size:
                remaining.add(wanted)
            else:
                logger.debug('... not the right size to be %s', wanted.name)

        if not remaining:
            for candidate in candidates:
                if not candidate.distinctive_name:
                    # silently ignore dissimilar file
                    logger.debug('... not a distinctive name, ignoring')
                    return False

            self._log_not_any_of(path, size, hashes, found, candidates)
            return False

        if hashes is None:
            hashes = HashedFile.from_file(path, open(path, 'rb'), size=size,
                    progress=self.progress_factory(info='checking %s' % path))

        for wanted in remaining:
            if not wanted.skip_hash_matching and not hashes.matches(wanted):
                logger.debug('... not the right hashes to be %s', wanted.name)
                continue

            if wanted.unsuitable:
                logger.warning('"%s" matches known file "%s" but cannot '
                        'be used:\n%s', path, wanted.name, wanted.unsuitable)
                # ... but do not continue processing
                return True

            logger.debug('... matches %s', wanted.name)
            self.found[wanted.name] = path
            self.file_status[wanted.name] = FillResult.COMPLETE

            # opportunistically use this same file to provide anything else that
            # has the same hashes (a duplicate file with a different name)
            for other_name in (self.game.known_md5s.get(hashes.md5, set()) |
                    self.game.known_sha1s.get(hashes.sha1, set()) |
                    self.game.known_sha256s.get(hashes.sha256, set())):
                other = self.game.files[other_name]
                if other is not wanted and other.matches(hashes):
                    logger.debug('... also matches %s', other_name)
                    self.found[other_name] = path
                    self.file_status[other_name] = FillResult.COMPLETE

            # no point in continuing, we've identified everything that matches
            # the hashes
            return True

        self._log_not_any_of(path, size, hashes, found, candidates)

    def consider_file(self, path, really_should_match_something, trusted=False):
        if not os.path.exists(path):
            # dangling symlink
            return

        match_path = '/' + path.lower()
        size = os.stat(path).st_size

        for p in self.game.rip_cd_packages:
            assert p.rip_cd

            # We use whatever the first track is (usually 2, because track
            # 1 is data) to locate the rest of the tracks.
            # We assume tracks in the middle are not missing.
            look_for = '/' + (p.rip_cd['filename_format'] %
                    p.rip_cd.get('first_track', 2))
            if match_path.endswith(look_for):
                self.cd_tracks.setdefault(p.name, {})
                # make sure it is at least as long as look_for
                # (corner-case: g-d-p quake id1/music)
                audio = path
                if not audio.startswith('/'):
                    audio = './' + audio
                basedir = audio[:len(audio) - len(look_for)]

                # The CD audio spec says we can't go beyond track 99.
                for i in range(p.rip_cd.get('first_track', 2),
                        p.rip_cd.get('last_track', 99) + 1):
                    track = p.rip_cd['filename_format'] % i
                    audio = os.path.join(basedir, track)
                    if not os.path.isfile(audio):
                        break
                    logger.debug(
                        'Found CD track %i for %s at %s', i, p.name, audio)
                    self._add_cd_track(p, i, audio)

                # Continue processing - maybe we can match it to a
                # known-good rip, which is useful information -
                # but don't warn if it doesn't match a known-good rip
                really_should_match_something = False

        # if a file (as opposed to a directory) is specified on the
        # command-line, try harder to match it to something
        if really_should_match_something:
            hashes = self.__ensure_hashes(None, path, size)
        else:
            hashes = None

        for look_for, candidates in self.game.known_filenames.items():
            if match_path.endswith('/' + look_for):
                candidates = [self.game.files[c] for c in candidates]
                if candidates:
                    hashes = self.__ensure_hashes(hashes, path, size)
                    if self.use_file('possible "%s"' % look_for, candidates,
                            path, hashes):
                        return

        if size in self.game.known_sizes:
            candidates = self.game.known_sizes[size]
            if candidates:
                hashes = self.__ensure_hashes(hashes, path, size)
                candidates = [self.game.files[c] for c in candidates]
                if self.use_file('file of size %d' % size,
                        candidates, path, hashes):
                    return

        if hashes is not None:
            look_for = None
            candidates = set()

            for c in (self.game.known_md5s.get(hashes.md5, set()) |
                    self.game.known_sha1s.get(hashes.sha1, set()) |
                    self.game.known_sha256s.get(hashes.sha256, set())):
                look_for = c
                candidates.add(self.game.files[c])

            if candidates and self.use_file('possible "%s"' % c,
                    candidates, path, hashes):
                return

            if not trusted:
                trusted = GOG.verify_checksum(path, size, hashes.md5)

        basename = os.path.basename(path)
        extension = os.path.splitext(basename)[1]
        if trusted:
            logger.warning('\n\nPlease report this unknown archive to '
                           'game-data-packager@packages.debian.org\n\n'
                           '  %-9s %s %s\n'
                           '  %s  %s\n' % (size, hashes.md5, basename, hashes.sha1, basename))
            if basename.startswith('gog_') and extension == '.sh':
                with ZipUnpacker(path) as unpacker:
                    self.consider_stream(path, unpacker)
            elif basename.startswith('setup_') and extension == '.exe':
                if not self.verbose:
                    logger.info('extracting %s (%d bytes) with InnoExtract...'
                                    % (basename, size))

                tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            basename + '.d')
                mkdir_p(tmpdir)

                with InnoSetup(path, verbose=self.verbose) as unpacker:
                    unpacker.extractall(tmpdir)

                self.consider_file_or_dir(tmpdir)
        elif really_should_match_something:
            logger.warning('file "%s" does not match any known file', path)
            # ... still G-D-P should try to process any random .zip
            # file thrown at it, like the .zip provided by GamersHell
            # or the MojoSetup installers provided by GOG.com
            if (extension.lower() in ('.zip', '.apk')
               or (basename.startswith('gog_') and extension == '.sh')):
                with ZipUnpacker(path) as unpacker:
                    self.consider_stream(path, unpacker)
            elif extension.lower() == '.deb' and which('dpkg-deb'):
                with subprocess.Popen(['dpkg-deb', '--fsys-tarfile', path],
                            stdout=subprocess.PIPE) as fsys_process:
                    with TarUnpacker(path + '//data.tar.*',
                           reader=fsys_process.stdout, compression='') as tar:
                        self.consider_stream(path, tar)

    def _log_not_any_of(self, path, size, hashes, why, candidates):
        message = ('found %s but it is not one of the expected ' +
                'versions:\n' +
                '    file:   %s\n' +
                '    size:   %d bytes\n' +
                '    md5:    %s\n' +
                '    sha1:   %s\n' +
                '    sha256: %s\n')
        args = (why, path, size, hashes.md5, hashes.sha1, hashes.sha256)

        candidates = [c for c in candidates if not c.unsuitable]

        if len(candidates) == 1:
            message += 'expected:\n'
        elif len(candidates) > 1:
            message += 'expected one of:\n'

        for candidate in candidates:
            message = message + ('  %s:\n' +
                    '    size:   ' + (
                        '%s' if candidate.size is None else '%d bytes') +
                    '\n' +
                    '    md5:    %s\n' +
                    '    sha1:   %s\n' +
                    '    sha256: %s\n')
            args = args + (candidate.name, candidate.size, candidate.md5,
                    candidate.sha1, candidate.sha256)

        logger.warning(message, *args)

    def consider_file_or_dir(self, path, provider=None):
        st = os.stat(path)

        if provider is None:
            should_provide = set()
        else:
            should_provide = set(provider.provides_files)

        if stat.S_ISREG(st.st_mode):
            self.consider_file(path, True)
        elif stat.S_ISDIR(st.st_mode):
            for dirpath, dirnames, filenames in os.walk(path):
                for fn in filenames:
                    self.consider_file(os.path.join(dirpath, fn), False)
        elif stat.S_ISBLK(st.st_mode):
            if self.game.rip_cd_packages:
                self.cd_device = path
            else:
                logger.warning('"%s" does not have a package containing CD '
                        'audio, ignoring block device "%s"',
                        self.game.shortname, path)
        else:
            logger.warning('file "%s" does not exist or is not a file, ' +
                    'directory or CD block device', path)

        for missing in sorted(f.name for f in should_provide):
            if missing not in self.found:
                logger.error('%s should have provided %s but did not',
                        self.found[provider.name], missing)

    def fill_gaps(self, package, download=False, log=True, recheck=False,
            requested=False):
        """Return a FillResult.
        """
        assert package is not None

        deactivated = False

        if requested:
            logger.debug('Package %s was specifically requested', package.name)
        elif package.activated_by_files:
            logger.debug(
                'Checking whether we have any interest in %s', package.name)
            # If the package has activated_by, we don't build it unless
            # either: we found one of the distinctive files by which it
            # is activated, or the user specifically asked for it.
            deactivated = True

            if package.rip_cd:
                cd_tracks = self.cd_tracks.get(package.name, {})

                if 'last_track' in package.rip_cd:
                    first_track = package.rip_cd.get('first_track', 2)
                    last_track = package.rip_cd['last_track']
                    filename_format = package.rip_cd['filename_format']

                    for i in range(first_track, last_track + 1):
                        install_as = filename_format % i

                        if install_as not in cd_tracks:
                            break
                    else:
                        logger.debug('... yes (all CD tracks found)')
                        deactivated = False
                elif cd_tracks:
                    logger.debug('... yes (CD tracks found, total unknown)')
                    deactivated = False

            for wanted in package.activated_by_files:
                logger.debug('Checking for %s', wanted.name)

                if wanted.name in self.found:
                    logger.debug('... yes')
                    deactivated = False
                    break
                else:
                    for alt in wanted.alternatives:
                        if alt in self.found:
                            logger.debug('... yes (%s)', alt)
                            deactivated = False
                            break

                    if not deactivated:
                        break

            if deactivated:
                logger.debug(
                    'No reason found to be interested in %s', package.name)
                return FillResult.DEACTIVATED

        logger.debug('trying to fill any gaps for %s', package.name)
        package_arch = self.package_architectures.get(package.name, 'all')

        if package.architecture == 'all':
            package_arch = 'all'
        elif package.architecture == 'any':
            package_arch = self.packaging.get_architecture()
            self.package_architectures[package.name] = package_arch
        else:
            archs = package.architecture.split()
            package_arch = self.packaging.get_architecture(
                package.architecture,
            )
            if package_arch in archs:
                self.package_architectures[package.name] = package_arch
            else:
                logger.warning('cannot produce "%s" on architecture %s',
                    package.name,
                    package_arch,
                )
                return FillResult.DEACTIVATED

        for wanted in package.install_files:
            if (
                wanted.architecture != 'all'
                and wanted.architecture != package_arch
            ):
                logger.debug(
                    '%s (arch:%s) not relevant for package %s (arch:%s)',
                    wanted.name, wanted.architecture,
                    package.name, package_arch,
                )
                self.file_status[wanted.name] = FillResult.DEACTIVATED
                continue

            if wanted.name not in self.found:
                for alt in wanted.alternatives:
                    if alt in self.found:
                        break
                else:
                    logger.debug('gap needs to be filled for %s: %s',
                            package.name, wanted.name)

        result = FillResult.COMPLETE

        if package.rip_cd and 'last_track' in package.rip_cd:
            first_track = package.rip_cd.get('first_track', 2)
            last_track = package.rip_cd['last_track']
            filename_format = package.rip_cd['filename_format']

            for i in range(first_track, last_track + 1):
                install_as = filename_format % i
                cd_tracks = self.cd_tracks.setdefault(package.name, {})

                if install_as in cd_tracks:
                    continue

                status = FillResult.IMPOSSIBLE

                for rip in package.rip_cd.get('known_rips', ()):
                    name = rip['filename_format'] % (
                        i + rip.get('offset', 0))

                    if name not in self.game.files:
                        continue

                    wanted = self.game.files[name]

                    self.fill_gap(
                        package, wanted, download=download,
                        recheck=recheck, log=log)
                    status |= self.file_status[name]

                    if name in self.found:
                        logger.debug(
                            'CD track %d for %s: using %s',
                            i, package.name, self.found[name])
                        self._add_cd_track(package, i, self.found[name])

                for other in self.game.rip_cd_packages:
                    if (other.rip_cd.get('reuse', {}).get('package', '') !=
                            package.name):
                        continue

                    for theirs, ours in other.rip_cd['reuse']['tracks'].items():
                        if ours != i:
                            continue

                        for rip in other.rip_cd.get('known_rips', ()):
                            name = rip['filename_format'] % (
                                theirs + rip.get('offset', 0))

                            if name not in self.game.files:
                                continue

                            wanted = self.game.files[name]
                            self.fill_gap(
                                package, wanted, download=download,
                                recheck=recheck, log=log)
                            status |= self.file_status[name]

                            if name in self.found:
                                logger.debug(
                                    'CD track %d for %s: using %s',
                                    i, package.name, self.found[name])
                                self._add_cd_track(
                                    package, i, self.found[name])

                logger.debug(
                    'CD track %d for %s: %s', i, package.name, status)
                result &= status
        elif package.rip_cd and not self.cd_tracks.get(package.name):
            logger.debug('no CD tracks found for %s', package.name)
            result = FillResult.IMPOSSIBLE
            return

        # search first for files that have only one provider,
        # to avoid extraneous downloads
        unique_provider = list()
        multi_provider = list()
        unimportant = list()
        for wanted in (package.install_files | package.optional_files):
            if self.file_status.get(wanted.name) is FillResult.DEACTIVATED:
                continue

            if wanted.doc:
                unimportant.append(wanted)
            elif len(self.game.providers.get(wanted.name,[])) == 1:
                unique_provider.append(wanted)
            else:
                multi_provider.append(wanted)

        for wanted in unique_provider + multi_provider + unimportant:
            if self.file_status.get(wanted.name) is FillResult.DEACTIVATED:
                continue

            if wanted.name not in self.found:
                # updates file_status as a side-effect
                self.fill_gap(package, wanted,
                        download=(download and wanted not in unimportant),
                        recheck=recheck,
                        log=(log and wanted in package.install_files))

            logger.debug('%s: %s', wanted.name, self.file_status[wanted.name])

            if wanted in package.install_files:
                # it is mandatory
                result &= self.file_status[wanted.name]

        for wanted in package.install_files:
            if self.file_status.get(wanted.name) is FillResult.DEACTIVATED:
                continue

            if wanted.name not in self.found:
                for alt in wanted.alternatives:
                    if alt in self.found:
                        break
                else:
                    logger.debug('unable to fill gap for %s: %s',
                            package.name, wanted.name)

        self.package_status[package.name] = result
        logger.debug('%s: %s', package.name, result)
        return result

    def consider_stream(self, name, unpacker, provider=None):
        if provider is None:
            try_to_unpack = self.game.files
            should_provide = set()
            distinctive_dirs = False
        else:
            try_to_unpack = set(f.name for f in provider.provides_files)
            should_provide = set(try_to_unpack)
            distinctive_dirs = provider.unpack.get('distinctive_dirs', True)

        for entry in unpacker:
            if not entry.is_extractable or not entry.is_regular_file:
                continue

            for filename in try_to_unpack:
                wanted = self.game.files.get(filename)

                if wanted is None:
                    continue

                if wanted.alternatives:
                    continue

                if wanted.size not in (None, entry.size):
                    continue

                match_path = '/' + entry.name.lower()

                for lf in wanted.look_for:
                    if not distinctive_dirs:
                        lf = os.path.basename(lf)

                    if match_path.endswith('/' + lf):
                        # use this one
                        break
                else:
                    # proceed to next entry
                    continue

                should_provide.discard(filename)

                if filename in self.found:
                    continue

                entryfile = unpacker.open(entry)

                tmp = os.path.join(self.get_workdir(),
                        'tmp', wanted.name)
                tmpdir = os.path.dirname(tmp)
                mkdir_p(tmpdir)

                wf = open(tmp, 'wb')

                hf = HashedFile.from_file(
                        name + '//' + entry.name, entryfile, wf,
                        size=entry.size,
                        progress=self.progress_factory(
                            info='extracting %s from %s' % (entry.name, name)),
                        )
                wf.close()

                if entry.mtime is not None:
                    orig_time = entry.mtime
                elif provider is not None:
                    orig_name = self.found[provider.name]
                    orig_time = os.stat(orig_name).st_mtime
                else:
                    orig_time = None

                if orig_time is not None:
                    os.utime(tmp, (orig_time, orig_time))

                if not self.use_file(wanted.name, (wanted,), tmp, hf):
                    os.remove(tmp)

        if should_provide:
            for missing in sorted(should_provide):
                logger.error('%s should have provided %s but did not',
                        name, missing)

    def cat_files(self, package, provider, wanted):
        other_parts = provider.unpack['other_parts']
        for p in other_parts:
            self.fill_gap(package, self.game.files[p], download=False, log=True)
            if p not in self.found:
                # can't concatenate: one of the bits is missing
                break
        else:
            # we didn't break, so we have all the bits
            path = os.path.join(self.get_workdir(), 'tmp',
                    wanted.name)
            mkdir_p(os.path.dirname(path))
            with open(path, 'wb') as writer:
                def open_files():
                    yield open(self.found[provider.name], 'rb')
                    for p in other_parts:
                        yield open(self.found[p], 'rb')

                hasher = HashedFile.from_concatenated_files(wanted.name,
                        open_files(), writer, size=wanted.size,
                        progress=self.progress_factory(info='building %s' %
                            wanted.name),
                        )
            orig_time = os.stat(self.found[provider.name]).st_mtime
            os.utime(path, (orig_time, orig_time))
            self.use_file(wanted.name, (wanted,), path, hasher)

    def fill_gap(self, package, wanted, download=False, log=True, recheck=False):
        """Try to unpack, download or otherwise obtain wanted.

        If download is true, we may attempt to download wanted or a
        file that will provide it.

        Return a FillResult.
        """
        if wanted.name in self.found:
            assert self.file_status[wanted.name] is FillResult.COMPLETE
            return FillResult.COMPLETE

        if self.file_status[wanted.name] is FillResult.IMPOSSIBLE and not recheck:
            return FillResult.IMPOSSIBLE

        if (self.file_status[wanted.name] is FillResult.DOWNLOAD_NEEDED and
                not download):
            return FillResult.DOWNLOAD_NEEDED

        logger.debug('could not find %s, trying to derive it...', wanted.name)

        self.file_status[wanted.name] = FillResult.IMPOSSIBLE

        if wanted.alternatives:
            for alt in wanted.alternatives:
                self.file_status[wanted.name] |= self.fill_gap(package,
                  self.game.files[alt], download=download, log=False,
                  recheck=recheck)
                if alt in self.found:
                    assert self.file_status[alt] is FillResult.COMPLETE
                    assert self.file_status[wanted.name] is FillResult.COMPLETE
                    return FillResult.COMPLETE

            if self.file_status[wanted.name] is FillResult.IMPOSSIBLE and log:
                logger.error('could not find a suitable version of %s:',
                        wanted.name)

                for alt in wanted.alternatives:
                    alt = self.game.files[alt]
                    logger.error('%s:\n' +
                            '  expected:\n' +
                            '    size:   ' + (
                                '%s' if alt.size is None else '%d bytes') +
                            '\n' +
                            '    md5:    %s\n' +
                            '    sha1:   %s\n' +
                            '    sha256: %s',
                            alt.name,
                            alt.size,
                            alt.md5,
                            alt.sha1,
                            alt.sha256)

            return self.file_status[wanted.name]

        # no alternatives: try getting the file itself

        if wanted.download:
            # we think we can get it
            self.file_status[wanted.name] = FillResult.DOWNLOAD_NEEDED

            if download:
                if self.downloader is None:
                    self.downloader = Downloader(
                        progress_factory=self.progress_factory)

                if self.save_downloads is not None:
                    dest = self.save_downloads
                else:
                    dest = self.get_workdir()

                try:
                    path, hasher = self.downloader.download(wanted, dest)
                except NotDownloadable:
                    # download() already issued a warning, do nothing
                    pass
                except OutOfSpace:
                    return FillResult.IMPOSSIBLE
                else:
                    if self.use_file(wanted.name, (wanted,), path, hasher):
                        assert self.found[wanted.name] == path
                        assert (self.file_status[wanted.name] ==
                                FillResult.COMPLETE)
                        return FillResult.COMPLETE
                    else:
                        # file corrupted or something
                        os.remove(path)

        providers = list(self.game.providers.get(wanted.name, ()))

        # pick smallest possible provider to download
        # example: this huge archive is a superset of the smaller one
        # 103M /var/www/html/ETQW-client-1.4-1.5-update.x86.run
        # 531M /var/www/html/ETQW-client-1.5-full.x86.run
        if len(providers) > 1:
            sizes = dict()
            for provider_name in providers:
                sizes[provider_name] = self.game.files[provider_name].size or 0
            providers = sorted(sizes, key=sizes.get)

        for provider_name in providers:
            provider = self.game.files[provider_name]

            # don't bother if we wouldn't be able to unpack it anyway
            if not self.check_unpacker(provider, log_as_warning=
                          self.file_status[wanted.name] is FillResult.IMPOSSIBLE):
                continue

            # recurse to unpack or (see whether we can) download the provider
            provider_status = self.fill_gap(package, provider,
                    download=download, log=log)

            # ... and it's other parts
            if (provider_status.is_possible
                and provider.unpack
                and 'other_parts' in provider.unpack):
                for p in provider.unpack['other_parts']:
                    part_status = self.fill_gap(package, self.game.files[p],
                                                download=False, log=log)
                    logger.debug('other part "%s" is %s' % (p, part_status))
                    provider_status &= part_status

            if provider_name in self.unpack_tried:
                logger.debug('already tried unpacking provider %s',
                        provider_name)
            elif provider_status is FillResult.COMPLETE:
                found_name = self.found[provider_name]
                logger.debug('trying provider %s found at %s',
                        provider_name, found_name)
                fmt = provider.unpack['format']

                self.unpack_tried.add(provider_name)

                if self.verbose and fmt in ('zip', 'unzip'):
                    with zipfile.ZipFile(found_name, 'r') as zf:
                        encoding = provider.unpack.get('encoding', 'cp437')
                        if zf.comment:
                            comment = zf.comment.decode(encoding, 'replace')
                            try:
                                print(comment)
                            except UnicodeError:
                                print(comment.encode('ascii', 'replace').decode('ascii'))
                        if 'FILE_ID.DIZ' in zf.namelist():
                            id_diz = ''
                            try:
                                entryfile = zf.open('FILE_ID.DIZ')
                                id_diz = entryfile.read().decode(encoding, 'replace')
                            except NotImplementedError:
                                if which('unzip'):
                                    id_diz = check_output(['unzip', '-c','-q',
                                        found_name, 'FILE_ID.DIZ']
                                        ).decode(encoding, 'replace')
                            try:
                                print(id_diz)
                            except UnicodeError:
                                print(id_diz.encode('ascii', 'replace').decode('ascii'))

                to_unpack = provider.unpack.get('unpack')

                if to_unpack is None:
                    to_unpack = []

                    for f in provider.provides_files:
                        to_unpack.append(f.name.split('?')[0])

                if fmt == 'dos2unix':
                    tmp = os.path.join(self.get_workdir(),
                            'tmp', wanted.name)
                    tmpdir = os.path.dirname(tmp)
                    mkdir_p(tmpdir)

                    rf = open(found_name, 'rb')
                    contents = rf.read()
                    wf = open(tmp, 'wb')
                    wf.write(contents.replace(b'\r\n', b'\n'))

                    orig_time = os.stat(found_name).st_mtime
                    os.utime(tmp, (orig_time, orig_time))
                    self.use_file(wanted.name, (wanted,), tmp, None)
                elif fmt in ('tar.*', 'tar.gz', 'tar.bz2', 'tar.xz'):
                    reader = open(found_name, 'rb')
                    with TarUnpacker(found_name, reader, compression=fmt[4:],
                            skip=provider.unpack.get('skip', 0)) as tar:
                        self.consider_stream(found_name, tar, provider)
                elif fmt == 'deb':
                    with subprocess.Popen(['dpkg-deb', '--fsys-tarfile', found_name],
                                stdout=subprocess.PIPE) as fsys_process:
                        with TarUnpacker(found_name + '//data.tar.*',
                                fsys_process.stdout, compression='') as tar:
                            self.consider_stream(found_name, tar, provider)
                elif fmt == 'zip':
                    if provider.name.startswith('gog_'):
                        package.used_sources.add(provider.name)
                    with ZipUnpacker(found_name) as unpacker:
                        self.consider_stream(found_name, unpacker, provider)
                elif fmt == 'lha':
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    arg = 'x' if self.verbose else 'xq'
                    # workaround for real LHa as seen in Fedora/RPMfusion
                    # that does not like seeing too many '?' or '.' in filenames
                    src = os.path.abspath(found_name)
                    if '?' in src:
                        newsrc = os.path.join(self.get_workdir(),
                                              os.path.basename(src).split('?')[0])
                        os.symlink(src, newsrc)
                        src = newsrc
                    check_call(['lha', arg, src] +
                            list(to_unpack),
                            cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'id-shr-extract':
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    check_call(['id-shr-extract', os.path.abspath(found_name)],
                            cwd=tmpdir)
                    # this format doesn't store a timestamp, so the extracted
                    # files will instead inherit the archive's timestamp
                    recursive_utime(tmpdir, os.stat(found_name).st_mtime)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'cabextract':
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    quiet = [] if self.verbose else ['-q']
                    check_call(['cabextract'] + quiet + ['-L',
                            os.path.abspath(found_name)], cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'unace-nonfree':
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    check_call(['unace', 'x',
                             os.path.abspath(found_name)] +
                             list(to_unpack), cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'unrar-nonfree':
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    quiet = [] if self.verbose else ['-inul']
                    check_call(['unrar-nonfree', 'x'] + quiet +
                             [os.path.abspath(found_name)] +
                             list(to_unpack), cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'innoextract':
                    if 'unpack' in provider.unpack:
                        to_unpack = provider.unpack['unpack']
                    else:
                        # this will result in extraneous "-I <file>" parameters,
                        # but innoextract doesn't care
                        to_unpack = set()
                        for f in provider.provides_files:
                            to_unpack.add(f.name.split('?')[0])
                            for l in f.look_for:
                                to_unpack.add(l)
                    to_unpack = sorted(to_unpack)
                    logger.debug('Extracting %r from %s', to_unpack, found_name)
                    package.used_sources.add(provider.name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                                          provider_name + '.d')
                    mkdir_p(tmpdir)

                    members = []

                    prefix = provider.unpack.get('prefix', '')

                    if prefix and not prefix.endswith('/'):
                        prefix += '/'

                    for i in to_unpack:
                        if prefix and i[0] != '/':
                            i = prefix + i

                        members.append(i)

                    with InnoSetup(
                        os.path.abspath(found_name),
                        verbose=self.verbose,
                        language=provider.unpack.get('language'),
                    ) as unpacker:
                        unpacker.extractall(tmpdir, members=members)

                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'unzip' and which('unzip'):
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    quiet = [] if self.verbose else ['-qq']
                    check_call(['unzip', '-j', '-C'] +
                                quiet + [os.path.abspath(found_name)] +
                            list(to_unpack), cwd=tmpdir)
                    # -j junk paths
                    # -C use case-insensitive matching
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt in ('7z', 'unzip'):
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    flags = provider.unpack.get('flags', [])
                    if not self.verbose:
                        flags.append('-bd')
                    check_call(['7z', 'x'] + flags +
                                [os.path.abspath(found_name)] +
                                list(to_unpack), cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt in ('unar', 'unzip'):
                    logger.debug('Extracting %r from %s', to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    quiet = [] if self.verbose else ['-q']
                    check_call(['unar', '-D'] +
                               quiet + [os.path.abspath(found_name)] +
                               list(to_unpack), cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'unshield':
                    logger.debug('Extracting %r from %s', to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                                          provider_name + '.d')
                    mkdir_p(tmpdir)
                    # we can't specify individual files to extract
                    # but we can narrow down to 'groups'
                    groups = provider.unpack.get('groups')
                    if groups:
                        # unshield only take last '-g' into account
                        for group in groups:
                            check_call(['unshield', '-g', group,
                               'x', os.path.abspath(found_name)], cwd=tmpdir)
                    else:
                        check_call(['unshield', 'x',
                                 os.path.abspath(found_name)], cwd=tmpdir)

                    # this format doesn't store a timestamp, so the extracted
                    # files will instead inherit the archive's timestamp
                    recursive_utime(tmpdir, os.stat(found_name).st_mtime)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'arj':
                    logger.debug('Extracting %r from %s',
                                 to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                                          provider_name + '.d')
                    mkdir_p(tmpdir)
                    check_call(['arj', 'e',
                                  os.path.abspath(found_name)] +
                                  list(to_unpack), cwd=tmpdir)
                    for p in provider.unpack.get('other_parts', []):
                        check_call(['arj', 'e', '-jya',
                                  os.path.join(os.path.dirname(found_name),p)] +
                                  list(to_unpack), cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)
                elif fmt == 'cat':
                    self.cat_files(package, provider, wanted)

                elif fmt in ('xdelta', 'xdelta3'):
                    # provider (found_name) is the delta
                    # other_parts contains only the base (unpatched) file
                    # wanted is the patched file
                    assert len(provider.unpack['other_parts']) == 1
                    basis = self.game.files[provider.unpack['other_parts'][0]]
                    if basis.name in self.found:
                        out_path = os.path.join(self.get_workdir(), 'tmp',
                                wanted.name)
                        mkdir_p(os.path.dirname(out_path))

                        if fmt == 'xdelta':
                            check_call(['xdelta', 'patch', found_name,
                                self.found[basis.name], out_path])
                        else:
                            check_call([
                                'xdelta3', '-d', '-s',
                                self.found[basis.name], found_name,
                                out_path])

                        orig_time = os.stat(found_name).st_mtime
                        os.utime(out_path, (orig_time, orig_time))
                        self.use_file(wanted.name, (wanted,), out_path)

                elif fmt == 'umod':
                    with Umod(found_name) as unpacker:
                        self.consider_stream(found_name, unpacker, provider)

                elif fmt == "xorriso":
                    logger.debug('Extracting %r from %s',
                            to_unpack, found_name)
                    tmpdir = os.path.join(self.get_workdir(), 'tmp',
                            provider_name + '.d')
                    mkdir_p(tmpdir)
                    check_call(['xorriso', '-osirrox', 'on', '-indev'] +
                                [os.path.abspath(found_name), '-extract'] +
                                list(to_unpack) + [tmpdir], cwd=tmpdir)
                    # we have to make the files in the tempdir writable
                    # otherwise the cleanup fails
                    check_call(['chmod', '-R', '+w', tmpdir], cwd=tmpdir)
                    self.consider_file_or_dir(tmpdir, provider=provider)                    

                if wanted.name in self.found:
                    assert (self.file_status[wanted.name] ==
                            FillResult.COMPLETE)
                    return FillResult.COMPLETE
            elif wanted.size == 0:
                self.use_file(wanted.name, (wanted,), '/dev/null')
            elif provider_status is FillResult.DOWNLOAD_NEEDED:
                # we don't have it, but we can get it
                self.file_status[wanted.name] |= FillResult.DOWNLOAD_NEEDED
            # else impossible, but try next provider

        if self.file_status[wanted.name] is FillResult.IMPOSSIBLE and log:
            logger.error('could not find %s:\n' +
                    '  expected:\n' +
                    '    size:   ' + (
                        '%s' if wanted.size is None else '%d bytes') +
                    '\n' +
                    '    md5:    %s\n' +
                    '    sha1:   %s\n' +
                    '    sha256: %s',
                    wanted.name,
                    wanted.size,
                    wanted.md5,
                    wanted.sha1,
                    wanted.sha256)

        return self.file_status[wanted.name]

    def check_complete(self, package, log=False):
        # Got everything?
        complete = True
        for wanted in package.install_files:
            if self.file_status.get(wanted.name) is FillResult.DEACTIVATED:
                continue

            if wanted.name in self.found:
                continue

            for alt in wanted.alternatives:
                if alt in self.found:
                    break
            else:
                complete = False
                if log:
                    logger.error('could not find %s:\n' +
                            '  expected:\n' +
                            '    size:   ' + (
                                '%s' if wanted.size is None else '%d bytes') +
                            '\n' +
                            '    md5:    %s\n' +
                            '    sha1:   %s\n' +
                            '    sha256: %s',
                            wanted.name,
                            wanted.size,
                            wanted.md5,
                            wanted.sha1,
                            wanted.sha256)

        return complete

    def __fill_docs(
        self,
        per_package_state: PerPackageState,
        pkgdocdir: str,
    ) -> None:
        package = per_package_state.package
        destdir = per_package_state.destdir

        copy_to = os.path.join(destdir, pkgdocdir.strip('/'), 'copyright')
        for n in (package.name, self.game.shortname):
            copy_from = os.path.join(DATADIR, n + '.copyright')
            if os.path.exists(copy_from):
                shutil.copyfile(copy_from, copy_to)
                return

            if os.path.exists(copy_from + '.in'):
                copy_with_substitutions(open(copy_from + '.in',
                            encoding='utf-8'),
                        open(copy_to, 'w', encoding='utf-8'),
                        PACKAGE=package.name)
                return

        copy_from = os.path.join(DATADIR, 'copyright')
        with open(copy_from, encoding='utf-8') as i, \
             open(copy_to, 'w', encoding='utf-8') as o:
            o.write('The package %s was generated using '
                    'game-data-packager.\n' % package.name)

            licenses = set()
            for f in (package.install_files | package.optional_files):
                if self.file_status[f.name] is not FillResult.COMPLETE:
                   continue
                if not f.license:
                    continue
                license_file = f.install_as
                licenses.add(os.path.join('/',
                    self.packaging.substitute('$pkglicensedir', package.name),
                    license_file))
                if os.path.splitext(license_file)[0].lower() == 'license':
                    per_package_state.lintian_overrides.add(
                        'extra-license-file usr/share/doc/{}/{}'.format(
                            package.name, license_file,
                        )
                    )

            if per_package_state.component == 'local':
                o.write('It contains proprietary game data '
                        'and must not be redistributed.\n\n')
            elif per_package_state.component == 'non-free':
                o.write('It contains proprietary game data '
                        'that may be redistributed\n'
                        'only under conditions specified in\n')
                o.write(',\n'.join(sorted(licenses)) + '.\n\n')
            else:
                o.write('It contains free game data and may be\n'
                        'redistributed under conditions specified in\n')
                o.write(',\n'.join(sorted(licenses)) + '.\n\n')


            notice = package.copyright_notice or self.game.copyright_notice
            if notice:
                 o.write('-' * 70)
                 o.write('\n\n' + notice + '\n')
                 o.write('-' * 70 + '\n\n')

            count_usr = 0
            exts = set()
            count_doc = 0
            for f in (package.install_files | package.optional_files):
                if self.file_status[f.name] is FillResult.IMPOSSIBLE:
                    continue
                install_to = f.install_to
                if install_to and install_to.startswith('$pkgdocdir'):
                    count_doc +=1
                elif install_to and install_to.startswith('$pkglicensedir'):
                    pass
                else:
                    count_usr +=1
                    # doesn't have to be a .wad, ROTT's EXTREME.RTL
                    # or any other one-datafile .deb would qualify too
                    main_wad = f.install_as
                    exts.add(os.path.splitext(main_wad.lower())[1])

            # XXX: this doesn't handle lgeneral or other externaly generated files
            if package.rip_cd:
                exts.add('.ogg')

            install_to = self.packaging.substitute(package.install_to,
                    package.name)

            if count_usr == 0 and count_doc == 1:
                o.write('"/usr/share/doc/%s/%s"\n' % (package.name,
                                                      package.only_file))
            elif count_usr == 1:
                o.write('"%s"\n' % os.path.join('/', install_to, main_wad))
            elif len(exts) == 1:
                o.write('The %s files under "%s/"\n' %
                        (list(exts)[0], os.path.join('/', install_to)))
            else:
                o.write('The files under "%s/"\n' % os.path.join('/', install_to))

            if count_usr and count_doc:
                if count_usr == 1:
                    o.write('and the files under "/usr/share/doc/%s/"\n' % package.name)
                else:
                    o.write('and "/usr/share/doc/%s/"\n' % package.name)
                o.write('(except for this copyright file & changelog.gz)\n')

            if (count_usr + count_doc) == 1:
                o.write('is a user-supplied file with copyright\n')
            else:
                o.write('are user-supplied files with copyright\n')

            o.write(package.copyright or self.game.copyright)
            o.write(', with all rights reserved.\n')

            if licenses and per_package_state.component == 'local':
                o.write('\nThe full license appears in ')
                o.write(',\n'.join(licenses))
                o.write('\n')

            for line in i.readlines():
                if line.startswith('#'):
                    continue
                o.write(line)


    def fill_extra_files(self, per_package_state: PerPackageState) -> None:
        pass

    def __fill_dest_dir(self, per_package_state: PerPackageState) -> None:
        package = per_package_state.package
        destdir = per_package_state.destdir

        install_to = self.packaging.substitute(
            package.install_to,
            package.name,
        )
        pkgdocdir = self.packaging.substitute('$pkgdocdir', package.name)
        dest_pkgdocdir = os.path.join(destdir, pkgdocdir.strip('/'))
        mkdir_p(dest_pkgdocdir)
        shutil.copyfile(os.path.join(DATADIR, 'changelog.gz'),
                os.path.join(dest_pkgdocdir, 'changelog.gz'))

        self.__check_component(per_package_state)
        self.__fill_docs(per_package_state, pkgdocdir)
        package_arch = self.package_architectures.get(package.name, 'all')

        for wanted in (package.install_files | package.optional_files):
            if (
                wanted.architecture != 'all'
                and wanted.architecture != package_arch
            ):
                continue

            install_as = wanted.install_as

            if wanted.name in self.found:
                copy_from = self.found[wanted.name]
                md5 = wanted.md5
            else:
                for alt in wanted.alternatives:
                    if alt in self.found:
                        copy_from = self.found[alt]
                        md5 = self.game.files[alt].md5
                        if wanted.install_as == '$alternative':
                            install_as = self.game.files[alt].install_as.split('?')[0]
                        break
                else:
                    if wanted not in package.install_files:
                        logger.debug('optional file %r is missing, ignoring',
                                wanted.name)
                        continue

                    raise AssertionError('we already checked that %s exists' %
                            (wanted.name))

            # cp it into place
            with TemporaryUmask(0o22):
                logger.debug('Found %s at %s', wanted.name, copy_from)

                if wanted.install_to is None:
                    file_install_to = install_to
                else:
                    file_install_to = self.packaging.substitute(
                        wanted.install_to,
                        package.name,
                        install_to=install_to,
                    )

                copy_to = os.path.join(
                    destdir,
                    file_install_to.strip('/'),
                    install_as,
                )
                assert copy_to.startswith(destdir + '/'), (copy_to, destdir)
                copy_to_dir = os.path.dirname(copy_to)
                logger.debug('Copying to %s', copy_to)
                if not os.path.isdir(copy_to_dir):
                    mkdir_p(copy_to_dir)
                # Use cp(1) so we can make a reflink if source and
                # destination happen to be the same btrfs volume
                subprocess.check_call(['cp', '--reflink=auto',
                    '--preserve=timestamps', copy_from, copy_to])

                if wanted.executable:
                    os.chmod(copy_to, 0o755)
                else:
                    os.chmod(copy_to, 0o644)

                fullname = os.path.join(file_install_to, install_as).strip('/')
                per_package_state.md5sums[fullname] = md5

        for symlink, real_file in package.symlinks.items():
            symlink = self.packaging.substitute(symlink, package.name,
                    install_to=install_to)
            real_file = self.packaging.substitute(real_file, package.name,
                    install_to=install_to)

            symlink = symlink.strip('/')
            real_file = real_file.strip('/')

            toplevel, rest = symlink.split('/', 1)
            if real_file.startswith(toplevel + '/'):
                symlink_dirs = symlink.split('/')
                real_file_dirs = real_file.split('/')

                while (len(symlink_dirs) > 0 and len(real_file_dirs) > 0 and
                        symlink_dirs[0] == real_file_dirs[0]):
                    symlink_dirs.pop(0)
                    real_file_dirs.pop(0)

                if len(symlink_dirs) == 0:
                    raise ValueError('Cannot create a symlink to itself')

                target = ('../' * (len(symlink_dirs) - 1)) + '/'.join(real_file_dirs)
            else:
                target = '/' + real_file

            mkdir_p(os.path.dirname(os.path.join(destdir, symlink)))
            os.symlink(target, os.path.join(destdir, symlink))

        if package.rip_cd and self.cd_tracks.get(package.name):
            for install_as, copy_from in self.cd_tracks[package.name].items():
                copy_to = os.path.join(destdir, install_to.strip('/'), install_as)

                if os.path.exists(copy_to):
                    continue

                logger.debug('Found CD track %s at %s', install_as, copy_from)
                assert copy_to.startswith(destdir + '/'), (copy_to, destdir)
                copy_to_dir = os.path.dirname(copy_to)
                if not os.path.isdir(copy_to_dir):
                    mkdir_p(copy_to_dir)
                check_call(['cp', '--reflink=auto',
                    '--preserve=timestamps', copy_from, copy_to])

        self.fill_extra_files(per_package_state)

    def look_for_engines(self, packages, force=False):
        engines = set()

        for p in packages:
            engines.add(self.packaging.substitute(p.engine or self.game.engine,
                    p.name))

        engines.discard(None)

        if not engines:
            return

        # XXX: handle complex cases too (e.g. Inherit the Earth DE vs EN)
        status = FillResult.UNDETERMINED
        for engine_alternative in engines:
            for engine in reversed(engine_alternative.split('|')):
                engine = engine.strip()
                status |= self.look_for_engine(engine)

        if status is FillResult.IMPOSSIBLE:
            if force:
                logger.warning('Engine "%s" is not available, '
                               'proceeding anyway' % engine)
            else:
                logger.error('Engine "%s" is not (yet) available, '
                             'aborting' % engine)
                raise SystemExit(1)
        elif status is FillResult.UPGRADE_NEEDED:
            if force:
                logger.warning('Engine "%s" is not up-to-date, '
                               'proceeding anyway' % engine)
            else:
                logger.error('Engine "%s" is not up-to-date, '
                             'aborting' % engine)
                raise SystemExit(1)

    def look_for_engine(self, engine):
        if '(' in engine:
            engine, ver = engine.split(maxsplit=1)
            ver = ver.strip('(>=) ')
        else:
            ver = None

        # check engine
        is_installed = self.packaging.is_installed(engine)
        if not is_installed and not self.packaging.is_available(engine):
            return FillResult.IMPOSSIBLE
        if ver is None:
            if is_installed:
                return FillResult.COMPLETE
            else:
                return FillResult.DOWNLOAD_NEEDED

        if self.packaging.available_version_at_least(
            engine,
            ver,
            is_installed=is_installed,
        ):
            return FillResult.COMPLETE
        else:
            return FillResult.UPGRADE_NEEDED

    def iter_extra_paths(self, packages):
        return []

    def look_for_files(self, paths=(), search=True, packages=None,
            binary_executables=False):
        paths = list(paths)

        if self.game.binary_executables:
            if not binary_executables:
                logger.error('%s requires binary-only executables which are '
                        'currently disallowed', self.game.longname)
                logger.info('Use the --binary-executables option to allow this, '
                        'at your own risk')
                raise BinaryExecutablesNotAllowed()

        if self.game.binary_executables and self.game.binary_executables != 'all':
            # 'all' means that this is well a binary without source,
            # but it can be emulated on any host architecture (e.g. DOSBox games)
            if self.packaging.get_architecture(self.game.binary_executables) not in \
                    self.game.binary_executables.split():
                logger.error('%s requires binary-only executables which are '
                        'only available for %s', self.game.longname,
                        ', '.join(self.game.binary_executables.split()))
                logger.info('If your CPU can run one of those architectures, '
                        'use dpkg --add-architecture to enable multiarch')
                raise NoPackagesPossible()

        if self.save_downloads is not None and self.save_downloads not in paths:
            paths.append(self.save_downloads)

        if packages is None:
            packages = self.game.packages.values()

        if search:
            for path in self.game.try_repack_from:
                path = os.path.expanduser(path)
                if os.path.isdir(path) and path not in paths:
                    paths.append(path)

            for package in packages:
                path = os.path.join('/',
                        self.packaging.substitute(package.install_to,
                            package.name).strip('/'))

                if os.path.isdir(path) and path not in paths:
                    paths.append(path)
                path = self.packaging.substitute('$pkgdocdir', package.name)
                if os.path.isdir(path) and path not in paths:
                    paths.append(path)

                if (self.packaging.__class__ is not
                        self.builder_packaging.__class__):
                    path = os.path.join('/',
                        self.builder_packaging.substitute(package.install_to,
                            package.name).strip('/'))
                    logger.debug('Maybe %s', path)

                    if os.path.isdir(path) and path not in paths:
                        paths.append(path)
                    path = self.builder_packaging.substitute('$pkgdocdir',
                            package.name)
                    logger.debug('Maybe %s', path)
                    if os.path.isdir(path) and path not in paths:
                        paths.append(path)

            for path in self.iter_steam_paths():
                if path not in paths:
                    paths.append(path)

            for path in self.iter_gog_paths():
                if path not in paths:
                    paths.append(path)

            for path in self.iter_origin_paths():
                if path not in paths:
                    paths.append(path)

            for path in self.iter_extra_paths(packages):
                if path not in paths:
                    paths.append(path)

        for arg in paths:
            logger.debug('%s...', arg)
            self.consider_file_or_dir(arg)

    def run_command_line(self, args):
        if logging.getLogger().isEnabledFor(logging.DEBUG):
            logger.debug('package description:\n%s',
                    yaml.dump(self.game.to_data(expand=False)))
            logger.debug('package description after expansion:\n%s',
                    yaml.dump(self.game.to_data(expand=True)))

        self.verbose = getattr(args, 'verbose', False)

        if self.packaging.__class__ is self.builder_packaging.__class__:
            preserve = (getattr(args, 'destination', None) is not None)
            install = getattr(args, 'install', True)
        else:
            preserve = True
            install = False
            if args.destination is None:
                raise SystemExit('Must specify a destination when '
                        'building packages for a different packaging '
                        'system')
            for tool in self.packaging.BUILD_DEP:
                if not which(tool) and not(self.packaging.is_installed(tool)):
                    logger.error('tool "%s" is needed to cross-build packages', tool)
                    self.missing_tools.add(tool)
            if self.missing_tools:
                self.log_missing_tools()
                raise SystemExit(1)

        if getattr(args, 'compress', None) is None:
            # default to not compressing if we aren't going to install it
            # anyway
            args.compress = preserve

        self.save_downloads = args.save_downloads

        for package in self.game.packages.values():
            if args.shortname in package.aliases:
                args.shortname = package.name
                break

        if (args.shortname != self.game.shortname and
                args.shortname in self.game.packages):
            if args.packages and args.packages != [args.shortname]:
                not_the_one = [p for p in args.packages if p != args.shortname]
                logger.error('--package="%s" is not consistent with '
                        'selecting "%s"', not_the_one, args.shortname)
                raise SystemExit(1)

            args.demo = True
            args.packages = [args.shortname]
            packages = set([self.game.packages[args.shortname]])
            requested_packages = packages
        elif args.packages:
            args.demo = True
            packages = set()
            for p in args.packages:
                if p not in self.game.packages:
                    logger.error('--package="%s" is not part of game '
                            '"%s"', p, args.shortname)
                    raise SystemExit(1)
                packages.add(self.game.packages[p])
            requested_packages = packages
        else:
            # if no packages were specified, we require --demo to build
            # a demo if we have its corresponding full game
            packages = set(self.game.packages.values())
            requested_packages = set()

        self.look_for_engines(packages, force=not args.install)

        try:
            self.look_for_files(paths=args.paths, search=args.search,
                    packages=packages,
                    binary_executables=args.binary_executables)
        except BinaryExecutablesNotAllowed:
            raise SystemExit(1)
        except NoPackagesPossible:
            raise SystemExit(1)

        try:
            ready = self.prepare_packages(packages,
                    build_demos=args.demo, download=args.download,
                    search=args.search, log_immediately=bool(args.packages),
                    everything=args.everything,
                    requested_packages=requested_packages)
        except NoPackagesPossible:
            logger.error('Unable to complete any packages.')
            if self.missing_tools:
                # we already logged warnings about the files as they came up
                self.log_missing_tools()
                raise SystemExit(1)

            # probably not enough files supplied?
            # print the help text, maybe that helps the user to determine
            # what they should have added
            if not os.environ.get('DEBUG') and not os.environ.get('GDP_DEBUG'):
                self.game.argument_parser.print_help()
            raise SystemExit(1)
        except DownloadNotAllowed:
            logger.error('Unable to complete any packages because ' +
                    'downloading missing files was not allowed.')
            self.log_missing_tools()
            raise SystemExit(1)
        except DownloadsFailed:
            # we already logged an error
            logger.error('Unable to complete any packages because downloads failed.')
            raise SystemExit(1)
        except CDRipFailed:
            logger.error('Unable to rip CD audio')
            raise SystemExit(1)

        if args.destination is None:
            destination = self.get_workdir()
        else:
            destination = args.destination

        pkgs = self.build_packages(ready,
                compress=getattr(args, 'compress', True),
                destination=destination)

        rm_rf(os.path.join(self.get_workdir(), 'tmp'))

        if preserve:
            for pkg in pkgs:
                print('generated "%s"' % os.path.abspath(pkg))

        if install:
            self.packaging.install_packages(
                pkgs,
                method=args.install_method,
                gain_root=args.gain_root_command,
                force=args.force_install,
            )

        engines_alt = set()

        for p in ready:
            engines_alt.add(self.packaging.substitute(p.engine or self.game.engine,
                    p.name))

        engines_alt.discard(None)
        engines = set()

        for engine_alt in engines_alt:
            for engine in reversed(engine_alt.split('|')):
                engine = engine.split('(')[0].strip()
                if self.packaging.is_installed(engine):
                    break
            else:
                engines.add(engine)

        if engines:
            print('it is recommended to also install this game engine: %s' % ', '.join(engines))

        if logger.isEnabledFor(logging.DEBUG) and which(self.packaging.CHECK_CMD):
            print('Now running %s...' % self.packaging.CHECK_CMD.title())
            for pkg in pkgs:
                subprocess.call([self.packaging.CHECK_CMD, pkg])

    def rip_cd(self, package):
        cd_device = self.cd_device
        if cd_device is None:
            cd_device = '/dev/cdrom'

        logger.info('Ripping CD tracks %d+ from %s for %s',
                package.rip_cd.get('first_track', 2), cd_device, package.name)

        assert package.rip_cd['encoding'] == 'vorbis', package.name
        for tool in ('cdparanoia', 'oggenc'):
            if which(tool) is None:
                logger.error('cannot rip CD "%s" for package "%s": ' +
                        '%s is not installed', cd_device, package.name,
                        tool)
                raise CDRipFailed()

        mkdir_p(os.path.join(self.get_workdir(), 'tmp'))
        tmp_wav = os.path.join(self.get_workdir(), 'tmp', 'rip.wav')

        self.cd_tracks[package.name] = {}

        for i in range(package.rip_cd.get('first_track', 2),
                package.rip_cd.get('last_track', 99) + 1):
            track = os.path.join(self.get_workdir(), 'tmp', '%d.ogg' % i)
            if subprocess.call(['cdparanoia', '-d', cd_device, str(i),
                    tmp_wav]) != 0:
                break
            check_call(['oggenc', '-o', track, tmp_wav])

            self._add_cd_track(package, i, track)

            if os.path.exists(tmp_wav):
                os.remove(tmp_wav)

        if not self.cd_tracks[package.name]:
            logger.error('Did not rip any CD tracks successfully for "%s"',
                    package.name)
            raise CDRipFailed()

    def _add_cd_track(self, package, number, filename):
        install_as = package.rip_cd['filename_format'] % number
        self.cd_tracks.setdefault(package.name, {})[install_as] = filename

        if 'reuse' in package.rip_cd:
            their_number = package.rip_cd['reuse']['tracks'].get(number)

            if their_number is not None:
                other_name = package.rip_cd['reuse']['package']
                logger.debug(
                    'Also using for %s track %d', other_name, their_number)
                other = self.game.packages[other_name]
                self._add_cd_track(other, their_number, filename)

    def prepare_packages(self, packages=None, build_demos=False, download=True,
            search=True, log_immediately=True, everything=False,
            requested_packages=()):
        if packages is None:
            packages = self.game.packages.values()

        possible = set()
        possible_with_lgogdownloader = set()
        possible_with_steamcmd = set()

        if self.cd_device is not None:
            rip_cd_packages = self.game.rip_cd_packages & packages
            if rip_cd_packages:
                if len(rip_cd_packages) > 1:
                    logger.error('cannot rip the same CD for more than one ' +
                            'music package, please specify one with ' +
                            '--package: %s',
                            ', '.join(sorted([p.name
                                for p in rip_cd_packages])))
                    raise CDRipFailed()
                self.rip_cd(rip_cd_packages.pop())

        for package in packages:
            gog_id = self.game.gog_download_name(package)
            steam_id = package.steam.get('id') or self.game.steam.get('id')
            if self.fill_gaps(package,
                    requested=(everything or package in requested_packages),
                    log=log_immediately) not in (FillResult.IMPOSSIBLE,
                        FillResult.DEACTIVATED):
                logger.debug('%s is possible', package.name)
                possible.add(package)
            # download game if it is already owned by user's GOG.com account
            # user must have used 'lgogdownloader' at least once to make this work
            elif gog_id and gog_id in GOG.owned_games():
                if which('innoextract') or GOG.is_native(gog_id):
                    if lang_score(package.lang) == 0:
                        logger.debug('%s can be downloaded with lgogdownloader', package.name)
                    else:
                        logger.info('%s can be downloaded with lgogdownloader', package.name)
                    possible.add(package)
                    possible_with_lgogdownloader.add(package.name)
                else:
                    self.missing_tools.add('innoextract')
            # don't download "http://steamcommunity.com/profiles/<steam_id>/games?xml=1"
            # if downloads are disabled
            elif steam_id and download and search and which('steamcmd'):
                # avoid import loop
                from .steam import (owned_steam_games,get_steam_account)
                for g in owned_steam_games():
                    if steam_id == g[0]:
                        logger.info('%s can be downloaded with steamcmd', package.name)
                        possible.add(package)
                        possible_with_steamcmd.add(package.name)
                        break
            else:
                logger.debug('%s is impossible', package.name)

        if not possible:
            logger.debug('No packages were possible')

            if log_immediately:
                # we already logged the errors so just give up
                raise NoPackagesPossible()

            # Repeat the process for the first (hopefully only)
            # demo/shareware package, so we can log its errors.
            for package in self.game.packages.values():
                if package.demo_for:
                    if self.fill_gaps(package=package,
                            log=True) not in (FillResult.IMPOSSIBLE,
                                FillResult.DEACTIVATED):
                        logger.error('%s unexpectedly succeeded on second ' +
                                'attempt. Please report this as a bug',
                                package.name)
                        possible.add(package)
                    else:
                        raise NoPackagesPossible()
            else:
                # If no demo, repeat the process for the first
                # (hopefully only) full package, so we can log *its* errors.
                for package in self.game.packages.values():
                    if package.type == 'full':
                        if self.fill_gaps(package=package,
                                requested=(
                                    everything or
                                    package in requested_packages),
                                log=True) not in (FillResult.IMPOSSIBLE,
                                    FillResult.DEACTIVATED):
                            logger.error('%s unexpectedly succeeded on ' +
                                    'second attempt. Please report this as '
                                    'a bug', package.name)
                            possible.add(package)
                        else:
                            raise NoPackagesPossible()
                else:
                    raise NoPackagesPossible()

        for package in set(possible):
            build_depends = self.packaging.merge_relations(package, 'build_depends')
            for tool in build_depends:
                tool = tool.strip()

                if not which(tool) and not self.builder_packaging.is_installed(tool):
                    logger.error('package "%s" is needed to build "%s"' %
                                 (tool, package.name))
                    possible.discard(package)
                    self.missing_tools.add(tool)

        logger.debug('possible packages: %r', set(p.name for p in possible))
        if not possible:
            raise NoPackagesPossible()

        # this fancy algorithm will be overiden by '--package' argument
        if not log_immediately:
            # this check is done before the language check to avoid to end up with
            # simon-the-sorcerer1-fr-data + simon-the-sorcerer1-dos-en-data
            for package in set(possible):
                for v in package.better_versions:
                    if self.game.packages[v] in possible:
                        logger.info('will not produce "%s" because better '
                                'version "%s" is also available',
                                package.name, v)
                        possible.discard(package)
                        break

            for package in set(possible):
                score = max(set(lang_score(l) for l in package.langs))
                if score == 0:
                    logger.info('will not produce "%s" '
                                'because "%s" is not in LANGUAGE selection',
                                package.name, package.lang)
                    possible.discard(package)
                    continue

                # keep only preferred language for this virtual package
                provides = self.packaging.merge_relations(package, 'provides')

                if provides:
                    for other_p in possible:
                        if other_p.name == package.name:
                            continue
                        other_provides = self.packaging.merge_relations(other_p,
                                'provides')
                        if other_provides - provides:
                            # it provides something this one doesn't
                            continue
                        if score < lang_score(other_p.lang):
                            logger.info('will not produce "%s" '
                                        'because "%s" is preferred language',
                                        package.name, other_p.lang)
                            possible.discard(package)
                            break
            if not possible:
                raise NoPackagesPossible()

        for package in set(possible):
            if (package.expansion_for
              and (package.expansion_for not in self.game.packages
                   or  self.game.packages[package.expansion_for] not in possible)
              and not self.packaging.is_installed(package.expansion_for)):
                for fullgame in possible:
                    if fullgame.type == 'full':
                        logger.warning("won't generate '%s' expansion, because "
                          'full game "%s" is neither available nor already installed;'
                          ' and we are packaging "%s" instead.',
                          package.name, package.expansion_for, fullgame.name)
                        possible.discard(package)
                        break
                else:
                  logger.warning('will generate "%s" expansion, but full game '
                     '"%s" is neither available nor already installed.',
                     package.name, package.expansion_for)

            if not build_demos and package.demo_for:
                for p in set(possible):
                    if p.type == 'full':
                        # no point in packaging a demo if we have any full
                        # version
                        logger.info('will not produce "%s" because we have '
                            'the full version "%s"', package.name, p.name)
                        possible.discard(package)
        if not possible:
            raise NoPackagesPossible()

        ready = set()
        lgogdownloaded = set()
        steam_password = None

        external_download = possible_with_lgogdownloader | possible_with_steamcmd
        for package in possible:
            logger.debug('will produce %s', package.name)
            result = self.fill_gaps(package=package, download=download,
              requested=(everything or package in requested_packages),
              log=package.name not in external_download,
              recheck=package.name in external_download)
            if result is FillResult.COMPLETE:
                ready.add(package)
            elif download and package.name in possible_with_lgogdownloader:
                gog_id = self.game.gog_download_name(package)
                if gog_id in lgogdownloaded:
                    # something went bad, G-D-P will complain a lot anyway
                    continue
                lgogdownloaded.add(gog_id)
                tmpdir = os.path.join(self.get_workdir(), gog_id)
                mkdir_p(tmpdir)
                try:
                    check_call(['lgogdownloader',
                                       '--download',
                                       '--include', 'installers',
                                       '--directory', tmpdir,
                                       '--subdir-game', '',
                                       '--platform', 'linux,windows',
                                       '--language', package.lang,
                                       '--game', '^' + gog_id + '$'])
                    # consider *.bin before the .exe file
                    main_archive = None
                    archives = []
                    for dirpath, dirnames, filenames in os.walk(tmpdir):
                        for fn in filenames:
                            archive = os.path.join(dirpath, fn)
                            archives.append(archive)
                            if os.path.splitext(fn)[1] in ('.exe', '.sh'):
                                main_archive = archive
                            else:
                                self.consider_file(archive, True, trusted=True)
                    if main_archive:
                        self.consider_file(main_archive, True, trusted=True)

                    # recheck file status
                    if self.fill_gaps(package, log=True, download=True,
                       requested=(everything or package in requested_packages),
                       recheck=True) not in (FillResult.IMPOSSIBLE,
                           FillResult.DEACTIVATED):
                       ready.add(package)
                    if self.save_downloads:
                        for archive in archives:
                            try:
                                shutil.move(archive, self.save_downloads)
                            except shutil.Error:
                                # file was already there, but not trusted
                                pass
                except subprocess.CalledProcessError:
                    pass
            elif package.name in possible_with_steamcmd:
                steam_id = package.steam.get('id') or self.game.steam.get('id')
                steam_account = get_steam_account()
                if not steam_password:
                    import getpass
                    steam_password = getpass.getpass("Please provided password"
                                                     " for Steam account %s:" %
                                                      steam_account)
                # this should be guessed automatically like for GOG
                # and not needed to be encoded in individual YAML files
                if package.steam.get('native') or self.game.steam.get('native'):
                    platform = []
                else:
                    platform = ['+@sSteamCmdForcePlatformType', 'windows']

                try:
                    check_call(['steamcmd'] + platform + [
                                '+login', steam_account, steam_password,
                                '+app_update', '%s' % steam_id, 'validate',
                                '+quit'])
                except subprocess.CalledProcessError:
                    pass
                for path in set(self.iter_steam_paths(packages=(package,))):
                    self.consider_file_or_dir(path)
                if self.fill_gaps(package, log=True, download=True,
                    requested=(everything or package in requested_packages),
                    recheck=True) not in (FillResult.IMPOSSIBLE,
                        FillResult.DEACTIVATED):
                    ready.add(package)
            elif (result is FillResult.DOWNLOAD_NEEDED or
                  package.name in possible_with_lgogdownloader) and not download:
                logger.warning('As requested, not downloading necessary ' +
                        'files for %s', package.name)
            else:
                logger.error('Failed to download necessary files for %s',
                        package.name)

        if not ready:
            if not download:
                raise DownloadNotAllowed()
            raise DownloadsFailed()

        logger.debug('packages ready for building: %r', set(p.name for p in ready))
        return ready

    def build_packages(self, ready, destination, compress):
        packages = set()

        for package in ready:
            if not self.check_complete(package, log=True):
                raise SystemExit(1)

            # TODO: Ideally we would keep package.name as-is, and only
            # put this modified name in the per_package_state: but we can't
            # do that until we have checked that all references to .name
            # after this point are using the right one
            package.name = package.name.split("?")[0]

            per_package_dir = os.path.join(
                self.get_workdir(),
                '%s_%s.d' % (
                    package.name,
                    self.package_architectures.get(package.name, 'all'),
                )
            )
            per_package_state = PerPackageState(package, per_package_dir)
            self.__fill_dest_dir(per_package_state)

            # only compress if the caller says we should, the YAML
            # says it's worthwhile, and this isn't a ripped CD
            # (Vorbis is already compressed)
            compression = compress and not package.rip_cd
            if compression:
                compression = self.game.compression

            pkg = self.packaging.build_package(
                per_package_state, self.game,
                destination, compress=compression,
            )
            assert pkg is not None
            packages.add(pkg)

        return packages

    def locate_steam_icon(self, package):
        id = package.steam.get('id') or self.game.steam.get('id')
        if not id:
            return
        for res in (128, 96, 64, 32):
            icon = '~/.local/share/icons/hicolor/%dx%d/apps/steam_icon_%d.png'
            icon = os.path.expanduser(icon % (res, res, id))
            if os.path.isfile(icon):
                logger.info('found icon provided by native Steam client at %s.' % icon)
                return icon
        return

    def iter_gog_paths(self, packages=None):
        if packages is None:
            packages = self.game.packages.values()

        dirnames = set()
        for p in list(packages) + [self.game]:
            if p.gog == False:
                continue
            # some games seem to list more than one installation path :-(
            path = p.gog.get('path')
            if isinstance(path, list):
                dirnames |= set(path)
            else:
                dirnames.add(path)
        dirnames.discard(None)
        if not dirnames:
            return

        for prefix in ('/opt/GOG Games', os.path.expanduser('~/GOG Games')):
            try:
                # We look for anything starting with an element of dirnames,
                # so that we'll pick up names like "Inherit The Earth German".
                for name in os.listdir(prefix):
                    for target in dirnames:
                        try:
                            if name.startswith(target):
                                path = os.path.join(prefix, name)
                                if os.path.isdir(path):
                                    logger.debug('possible %r found at %r',
                                            self.game.shortname, path)
                                    yield os.path.realpath(path)
                        except OSError:
                            continue
            except OSError:
                continue

    def iter_steam_paths(self, packages=None):
        if packages is None:
            packages = self.game.packages.values()

        suffixes = set()
        for p in packages:
            if isinstance(p.steam.get('path'), list):
                for path in p.steam.get('path'):
                    suffixes.add(path)
            else:
                suffixes.add(p.steam.get('path'))
        if isinstance(self.game.steam.get('path'), list):
            for path in self.game.steam.get('path'):
                suffixes.add(path)
        else:
            suffixes.add(self.game.steam.get('path'))
        suffixes.discard(None)
        if not suffixes:
            return

        for prefix in (
                os.path.expanduser('~/.steam'),
                os.path.join(os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')),
                    'wineprefixes/steam/drive_c/Program Files/Steam'),
                os.path.join(os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')),
                    'wineprefixes/steam/drive_c/Program Files (x86)/Steam'),
                os.path.expanduser('~/Steam'),
                os.path.expanduser('~/.wine/drive_c/Program Files/Steam'),
                os.path.expanduser('~/.wine/drive_c/Program Files (x86)/Steam'),
                os.path.expanduser('~/.PlayOnLinux/wineprefix/Steam/drive_c/Program Files/Steam'),
                ) + tuple(iter_fat_mounts('Steam')):
            if not os.path.isdir(prefix):
                continue

            logger.debug('possible Steam root directory at %s', prefix)

            for middle in ('steamapps', 'steam/steamapps', 'SteamApps',
                    'steam/SteamApps', 'debian-installation/steamapps'):
                for suffix in suffixes:
                    path = os.path.join(prefix, middle, suffix)
                    if os.path.isdir(path):
                        logger.debug('possible %s found in Steam at %s',
                                self.game.shortname, path)
                        yield os.path.realpath(path)

    def iter_origin_paths(self, packages=None):
        if packages is None:
            packages = self.game.packages.values()

        suffixes = set(p.origin.get('path') for p in packages)
        suffixes.add(self.game.origin.get('path'))
        suffixes.discard(None)
        if not suffixes:
            return

        for prefix in (
                os.path.expanduser('~/.wine/drive_c/Program Files/Origin Games'),
                ) + tuple(iter_fat_mounts('Origin Games')):
            if not os.path.isdir(prefix):
                continue

            logger.debug('possible Origin root directory at %s', prefix)

            for suffix in suffixes:
                path = os.path.join(prefix, suffix)
                if os.path.isdir(path):
                    logger.debug('possible %s found in Origin at %s',
                            self.game.shortname, path)
                    yield path

    def __check_component(self, per_package_state: PerPackageState) -> None:
        # redistributable packages are redistributable as long as their
        # optional license file is present
        if per_package_state.component == 'local':
            return
        for f in per_package_state.package.optional_files:
             if not f.license:
                 continue

             if self.file_status[f.name] is not FillResult.COMPLETE:
                 per_package_state.component = 'local'
                 return
        return

    def check_unpacker(self, wanted, log_as_warning):
        if not wanted.unpack:
            return True

        if wanted.name in self.unpack_tried:
            return False

        fmt = wanted.unpack['format']

        # builtins
        if fmt in ('cat', 'dos2unix', 'tar.*', 'tar.gz', 'tar.bz2', 'tar.xz', 'umod', 'zip'):
            return True

        if fmt == 'deb':
            fmt = 'dpkg-deb'

        if which(fmt) is not None:
            return True

        if fmt == 'unzip' and (which('7z') or which('unar')):
            return True

        # unace-nonfree package diverts /usr/bin/unace from unace package
        if (fmt == 'unace-nonfree' and
                self.builder_packaging.is_installed('unace-nonfree')):
            return True


        if log_as_warning:
            logger.warning('cannot unpack "%s": tool "%s" is not ' +
                           'installed', wanted.name, fmt)
        else:
            logger.info('cannot unpack "%s": tool "%s" is not ' +
                        'installed', wanted.name, fmt)

        self.missing_tools.add(fmt)
        self.unpack_tried.add(wanted.name)
        return False

    def log_missing_tools(self):
        if not self.missing_tools:
            return False

        packages = set()

        for t in self.missing_tools:
            p = self.builder_packaging.package_for_tool(t)
            if p is not None:
                packages.add(p)

        if packages:
            logger.warning('installing these packages might help:\n' +
                '%s %s', ' '.join(self.builder_packaging.INSTALL_CMD),
                ' '.join(sorted(packages)))

    def __ensure_hashes(self, hashes, path, size):
        if hashes is not None:
            return hashes

        return HashedFile.from_file(path, open(path, 'rb'), size=size,
                progress=self.progress_factory(info='identifying %s' % path))
