# Fetch pool files.
#
# The script 40phase-cd.bash and the script you are reading are the
# program's heart.
#
# The script modifies the following, intialized in 07initialization.bash.
# They carry through to the rest of the program:
#
#     DIST1 MIRROR
#     IS_BACKPORTS (integer)
#
# The script writes several files.  Other scripts ignore the files.  It
# would not exactly harm anything if other scripts overwrote or deleted
# the files, but since a debugging maintainer or user might be
# interested in them, the other scripts should leave these alone:
#
#     {regular,backports}-filepaths{,2}.list
#     backports-{filepaths-rp,missing}.list
#     {must,may}-deliver.list
#
# Further variables or files this script writes are of local interest
# only.  Other scripts may delete or overwrite.

readonly MSGE10=$(gettext 'fetching pool files')
readonly MSGE40=$(gettext 'the last scan selects these')
readonly MSGE90=$(gettext 'pool files fetched')
readonly PHE_URL="https://$SNAPSHOT_SERVICE/$ARCHIVE_NAME/$SNAPSHOT_SSDATE"

declare CTRL_DIR AREA0 AREA FP_FROM_SHA DIR CTRL CTRL1
declare -i IS_IN_POOL
if (($OPT_PHASE_E)); then

    ((!$OPT_SUMS)) && inform "$MSGE10..."
    pushd >/dev/null "$TARGET"

    # List data for actions to take later.
    for DIST1 in\
        "$DIST" $( (($OPT_BACKPORTS)) && printf '%s' "$DIST_BACKPORTS" )
    do

        # Appraise whether backports.
        IS_BACKPORTS=1; [ "$DIST1" = "$DIST" ] && IS_BACKPORTS=0

        # The following code is a bit of a spaghetti in how the various
        # filepath*.list files are made and deleted. C++ RAII would have
        # helped.

        # Extract filepaths from control files.
        #
        # If $OPT_SUMS, the following implcitly assumes that dists/ harbors
        # no irrelevant files named "Sources", "Packages" or "SHA256SUMS".
        # The assumption seems safe enough, but better software engineering
        # would probably avoid the assumption.
        touch -- $TMPDIR/filepaths0.list
        for CTRL1 in\
            $("$FINDX" '' '-name Sources' "dists/$DIST1")
        do
            CTRL=${CTRL1#./}
            ((!$OPT_SUMS)) && inform_of_scan "$CTRL"
            AREA0=${CTRL#$(printf '%q' "dists/$DIST1")/}
            AREA=${AREA0%%/*}
            filepaths_from_sources_ctrl "$CTRL" "$AREA"\
             >>$TMPDIR/filepaths0.list
        done
        for CTRL1 in\
            $("$FINDX" '' '-name Packages' "dists/$DIST1")
        do
            CTRL=${CTRL1#./}
            ((!$OPT_SUMS)) && inform_of_scan "$CTRL"
            filepaths_from_packages_ctrl "$CTRL"\
             >>$TMPDIR/filepaths0.list
        done
        for CTRL1 in\
            $("$FINDX" '' '-name SHA256SUMS' "dists/$DIST1")
        do
            CTRL=${CTRL1#./}
            FP_FROM_SHA=$(mktemp)
            # The next is a BASE6. (See the notes at the head
            # of 10function-definitions.bash.)
            CTRL_DIR=$(dirname "${CTRL#./}")
            filepaths_from_sha_ctrl "$CTRL" "$CTRL_DIR"\
             >>"$FP_FROM_SHA"
            if ((!$OPT_SUMS)); then
                inform_of_scan "$CTRL"
                inform "$MSGE40:"
                cat -- "$FP_FROM_SHA"
                print_bar
            fi
            cat >>$TMPDIR/filepaths0.list -- "$FP_FROM_SHA"
            rm -- "$FP_FROM_SHA"
        done

        if ((!$OPT_SUMS)); then
            # Sort filepaths into locally available and otherwise.
            touch -- $TMPDIR/filepaths{,2}.list
            (($IS_BACKPORTS)) && touch --\
             $TMPDIR/backports-filepaths-rp.list
            for FP1 in $(<$TMPDIR/filepaths0.list); do
                # Note that locally available files outside the pool could
                # not be straightforwardly copied by this phase's method
                # because a straightforward copy would mishandle directory
                # links (which the pool fortunately lacks). This phase's
                # method depends on timestamps of files in the pool not to
                # change, anyway, whereas dists though much smaller is more
                # complex. The maintainer is less confident of timestamping
                # practices in dists. For simplicity and reliability,
                # therefore, at the cost of some strictly unnecessary data
                # transfers over the network during a successive execution
                # of the script (but only for the small dists, not for the
                # big pool) files of dists are redownloaded during
                # each execution.
                IS_IN_POOL=0; [[ "$FP1" = pool/* ]] && IS_IN_POOL=1
                if (($IS_IN_POOL)) && [ -f "$OLD/$FP1" ]; then
                    printf >>$TMPDIR/filepaths2.list '%s\n' "$FP1"
                else
                    printf >>$TMPDIR/filepaths.list '%s\n' "$FP1"
                    (($IS_BACKPORTS)) && (($IS_IN_POOL))\
                     && printf >>$TMPDIR/backports-filepaths-rp.list\
                     '%s\n' "$FP1"
                fi
            done
        fi

        # Save the paths files.
        if (($OPT_SUMS)); then
            if (($IS_BACKPORTS)); then
                mv $VV -- $TMPDIR/{filepaths0,backports-filepaths}.list
                touch -- $TMPDIR/backports-filepaths2.list
            else
                mv $VV -- $TMPDIR/{filepaths0,regular-filepaths}.list
                touch -- $TMPDIR/regular-filepaths2.list
            fi
        else
            rm $VV -- $TMPDIR/filepaths0.list
            if (($IS_BACKPORTS)); then
                mv $VV -- $TMPDIR/{,backports-}filepaths.list
                mv $VV -- $TMPDIR/{,backports-}filepaths2.list
            else
                mv $VV -- $TMPDIR/{,regular-}filepaths.list
                mv $VV -- $TMPDIR/{,regular-}filepaths2.list
            fi
        fi

    done

    # At this point, the following lists exist:
    #
    #   + regular-filepaths.list
    #   + regular-filepaths2.list
    #   + backports-filepaths.list
    #   + backports-filepaths2.list
    #
    # These four lists are exclusive. The *-filepaths2.list (there are two)
    # list pool files that can be copied locally rather than retrieved.
    # (If $OPT_SUMS, then the *-filepaths2.list are empty.) All nonpool
    # files go to *-filepaths.list: no local copying.
    #
    # If !$OPT_SUMS, then this list exists, too:
    #
    #   + backports-filepaths-rp.list
    #
    # The -rp stands for "rsyncable pool". This list is nonexclusive, but
    # lists the subset of backports-filepaths.list the script may try to
    # retrieve from the primary mirror before falling back to rely on
    # snapshot. All files the backports-filepaths-rp.list lists are
    # pool files.
    #
    # On the other hand, if $OPT_SUMS, then you have
    #
    #   + ctrl-filepaths.list
    #
    # from phase D in 40phase-cd.bash.

    if (($OPT_SUMS)); then

        # Sort and print checksums.
        if (($OPT_BACKPORTS)); then
            sort_by_filename $TMPDIR/{ctrl,regular,backports}-filepaths.list
        else
            sort_by_filename $TMPDIR/{ctrl,regular}-filepaths.list
        fi

    else

        # Fetch remote files (and copy locally available files).

        # Copy locally available regular pool files.
        copy_locally_available $TMPDIR/regular-filepaths2.list

        # Fetch regular files, including nonpool files, the primary mirror
        # must deliver. (Note that, for files already locally present, the
        # primary mirror must still refresh the local timestamps.)
        touch -- $TMPDIR/must-deliver.list
        cat >$TMPDIR/must-deliver.list --\
         $TMPDIR/regular-filepaths{,2}.list
        [ -s $TMPDIR/must-deliver.list ]\
         && run_by_batch --persist rsync '-lKtRv --progress --stats --'\
        <(
            {
                while read; do
                    printf '%s\n' "::$ARCHIVE_NAME/$REPLY"
                done
            } <$TMPDIR/must-deliver.list
        ) ./ "$MIRROR_PRIMARY"

        if (($OPT_BACKPORTS)); then

            # Copy locally available backports pool files.
            copy_locally_available $TMPDIR/backports-filepaths2.list

            # Fetch backports files the primary mirror may deliver. (Unlike
            # the earlier, must-deliver files, these may-deliver files are
            # all pool files.)
            touch -- $TMPDIR/may-deliver.list
            cat >$TMPDIR/may-deliver.list --\
             $TMPDIR/backports-filepaths{-rp,2}.list
            [ -s $TMPDIR/may-deliver.list ]\
             && run_by_batch --persist rsync\
             '-lKtRv --ignore-missing-args --progress --stats --'\
            <(
                {
                    while read; do
                        printf '%s\n' "::$ARCHIVE_NAME/$REPLY"
                    done
                } <$TMPDIR/may-deliver.list
            ) ./ "$MIRROR_PRIMARY"

            # From snapshot, fetch backports files, including nonpool
            # files, that are still missing. (Because wget rather than
            # rsync is used, timestamps of files already locally present
            # are not touched.)
            touch -- $TMPDIR/backports-missing.list
            for FP1 in $(<$TMPDIR/backports-filepaths.list); do
                [ -e "$FP1" ] || printf\
                >>$TMPDIR/backports-missing.list '%s\n' "$FP1"
            done
            # (Note: the double-escape with quotes a few lines below
            # is necessary, for the string gets interpolated twice.)
            [ -s $TMPDIR/backports-missing.list ]\
             && run_by_batch wget_insist\
             "'"'-nc -x -nH --cut-dirs=4 -P ./pool'"'"\
            <(
                { while read; do printf '%s\n' "$PHE_URL/$REPLY"; done; }\
                 <$TMPDIR/backports-missing.list
            ) ''
        fi

    fi

    popd >/dev/null
    ((!$OPT_SUMS)) && inform "$MSGE90"

fi

true

