# Copyright 2019-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import functools
import io
import multiprocessing
import os
import shlex
import shutil
import signal
import tempfile
import types

from _emerge.EbuildFetcher import EbuildFetcher
from _emerge.Package import Package

import portage
from portage._emirrordist.Config import Config as EmirrordistConfig
from portage.checksum import checksum_str
from portage.const import BASH_BINARY, MANIFEST2_HASH_DEFAULTS, PORTAGE_PYM_PATH
from portage.package.ebuild.config import config
from portage.package.ebuild.digestgen import digestgen
from portage.package.ebuild.fetch import (
    ContentHashLayout,
    DistfileName,
    FilenameHashLayout,
    FlatLayout,
    MirrorLayoutConfig,
    _download_suffix,
    fetch,
)
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import ResolverPlayground
from portage.tests.util.test_socks5 import AsyncHTTPServer
from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.configparser import ConfigParserError
from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor


class EbuildFetchTestCase(TestCase):

    async def _test_interrupt(self, loop, server, async_fetch, pkg, ebuild_path):
        """Test interrupt, with server responses temporarily paused."""
        server.pause()
        pr, pw = multiprocessing.Pipe(duplex=False)
        timeout = loop.create_future()
        loop.add_reader(pr.fileno(), lambda: timeout.done() or timeout.set_result(None))
        self.assertEqual(
            await async_fetch(
                pkg,
                ebuild_path,
                timeout=timeout,
                pre_exec=functools.partial(self._pre_exec_interrupt_patch, pw),
            ),
            -signal.SIGTERM,
        )
        loop.remove_reader(pr.fileno())
        pw.close()

        # Read pid written by _async_spawn_fetch_pre_wait hook (the
        # corresponding write served to trigger the timeout above).
        pid = pr.recv()

        # Read pid written by _async_spawn_fetch_post_terminate hook,
        # in order to know when the ProcessLookupError test should
        # succeed.
        pid = pr.recv()
        pr.close()

        # Poll the process table until the pid has disappeared,
        # and fail if a short timeout expires.
        tries = 10
        while tries:
            tries -= 1

            msg = None
            if tries <= 0:
                try:
                    with open(f"/proc/{pid}/status") as f:
                        for line in f:
                            if line.startswith("State:"):
                                msg = line
                                break
                except OSError:
                    pass

            try:
                with self.assertRaises(ProcessLookupError, msg=msg):
                    os.kill(pid, 0)
            except Exception:
                if tries <= 0:
                    raise
                await asyncio.sleep(0.1)
            else:
                break

        server.resume()

    @staticmethod
    def _pre_exec_interrupt_patch(pw):
        portage.package.ebuild.fetch._async_spawn_fetch_pre_wait = functools.partial(
            EbuildFetchTestCase._fetch_pre_wait,
            pw,
        )
        portage.package.ebuild.fetch._async_spawn_fetch_post_terminate = (
            functools.partial(
                EbuildFetchTestCase._fetch_post_terminate,
                pw,
            )
        )

    @staticmethod
    def _fetch_pre_wait(pw, proc):
        pw.send(proc.pid)

    @staticmethod
    def _fetch_post_terminate(pw, proc):
        pw.send(proc.pid)

    def testEbuildFetch(self):
        user_config = {
            "make.conf": ('GENTOO_MIRRORS="{scheme}://{host}:{port}"',),
        }

        distfiles = {
            "bar": b"bar\n",
            "foo": b"foo\n",
        }

        ebuilds = {
            "dev-libs/A-1": {
                "EAPI": "7",
                "SRC_URI": """{scheme}://{host}:{port}/distfiles/bar.txt -> bar
					{scheme}://{host}:{port}/distfiles/foo.txt -> foo""",
            },
        }

        loop = SchedulerInterface(global_event_loop())

        scheme = "http"
        host = "127.0.0.1"
        content = {}

        with AsyncHTTPServer(host, content, loop) as server:
            ebuilds_subst = {}
            for cpv, metadata in ebuilds.items():
                metadata = metadata.copy()
                metadata["SRC_URI"] = metadata["SRC_URI"].format(
                    scheme=scheme, host=host, port=server.server_port
                )
                ebuilds_subst[cpv] = metadata

            user_config_subst = user_config.copy()
            for configname, configdata in user_config.items():
                configdata_sub = []
                for line in configdata:
                    configdata_sub.append(
                        line.format(scheme=scheme, host=host, port=server.server_port)
                    )
                user_config_subst[configname] = tuple(configdata_sub)

            playground = ResolverPlayground(
                ebuilds=ebuilds_subst,
                distfiles=distfiles,
                user_config=user_config_subst,
            )
            ro_distdir = tempfile.mkdtemp()
            try:
                self._testEbuildFetch(
                    loop,
                    scheme,
                    host,
                    distfiles,
                    ebuilds,
                    content,
                    server,
                    playground,
                    ro_distdir,
                )
            finally:
                shutil.rmtree(ro_distdir)
                playground.cleanup()

    def _testEbuildFetch(
        self,
        loop,
        scheme,
        host,
        orig_distfiles,
        ebuilds,
        content,
        server,
        playground,
        ro_distdir,
    ):
        mirror_layouts = (
            (
                "[structure]",
                "0=filename-hash BLAKE2B 8",
                "1=flat",
            ),
            (
                "[structure]",
                "1=filename-hash BLAKE2B 8",
                "0=flat",
            ),
            (
                "[structure]",
                "0=content-hash SHA512 8:8:8",
                "1=flat",
            ),
        )

        fetchcommand = shlex.split(playground.settings["FETCHCOMMAND"])
        fetch_bin = portage.process.find_binary(fetchcommand[0])
        if fetch_bin is None:
            self.skipTest(
                f"FETCHCOMMAND not found: {playground.settings['FETCHCOMMAND']}"
            )
        eubin = os.path.join(playground.eprefix, "usr", "bin")
        os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin)))
        resumecommand = shlex.split(playground.settings["RESUMECOMMAND"])
        resume_bin = portage.process.find_binary(resumecommand[0])
        if resume_bin is None:
            self.skipTest(
                f"RESUMECOMMAND not found: {playground.settings['RESUMECOMMAND']}"
            )
        if resume_bin != fetch_bin:
            os.symlink(resume_bin, os.path.join(eubin, os.path.basename(resume_bin)))
        root_config = playground.trees[playground.eroot]["root_config"]
        portdb = root_config.trees["porttree"].dbapi

        def run_async(func, *args, **kwargs):
            with ForkExecutor(loop=loop) as executor:
                return loop.run_until_complete(
                    loop.run_in_executor(
                        executor, functools.partial(func, *args, **kwargs)
                    )
                )

        for layout_lines in mirror_layouts:
            settings = config(clone=playground.settings)
            layout_data = "".join(f"{line}\n" for line in layout_lines)
            mirror_conf = MirrorLayoutConfig()
            mirror_conf.read_from_file(io.StringIO(layout_data))
            layouts = mirror_conf.get_all_layouts()
            content["/distfiles/layout.conf"] = layout_data.encode("utf8")
            distfiles = {}
            for k, v in orig_distfiles.items():
                filename = DistfileName(
                    k,
                    digests={
                        algo: checksum_str(v, hashname=algo)
                        for algo in MANIFEST2_HASH_DEFAULTS
                    },
                )
                distfiles[filename] = v

                # mirror path
                for layout in layouts:
                    content["/distfiles/" + layout.get_path(filename)] = v
                # upstream path
                content[f"/distfiles/{k}.txt"] = v

            shutil.rmtree(settings["DISTDIR"])
            os.makedirs(settings["DISTDIR"])
            with open(os.path.join(settings["DISTDIR"], "layout.conf"), "w") as f:
                f.write(layout_data)

            if any(isinstance(layout, ContentHashLayout) for layout in layouts):
                content_db = os.path.join(
                    playground.eprefix, "var/db/emirrordist/content.db"
                )
                os.makedirs(os.path.dirname(content_db), exist_ok=True)
                try:
                    os.unlink(content_db)
                except OSError:
                    pass
            else:
                content_db = None

            # Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given.
            foo_uri = {
                "foo": (f"{scheme}://{host}:{server.server_port}/distfiles/foo",)
            }
            foo_path = os.path.join(settings["DISTDIR"], "foo")
            foo_stale_content = b"stale content\n"
            with open(foo_path, "wb") as f:
                f.write(b"stale content\n")

            self.assertTrue(
                bool(run_async(fetch, foo_uri, settings, try_mirrors=False))
            )

            with open(foo_path, "rb") as f:
                self.assertEqual(f.read(), foo_stale_content)
            with open(foo_path, "rb") as f:
                self.assertNotEqual(f.read(), distfiles["foo"])

            # Use force=True to update the stale file.
            self.assertTrue(
                bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))
            )

            with open(foo_path, "rb") as f:
                self.assertEqual(f.read(), distfiles["foo"])

            # Test force=True with FEATURES=skiprocheck, using read-only DISTDIR.
            # FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that
            # FETCHCOMMAND must perform atomic rename itself due to read-only
            # DISTDIR.
            with open(foo_path, "wb") as f:
                f.write(b"stale content\n")
            orig_fetchcommand = settings["FETCHCOMMAND"]
            orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode
            temp_fetchcommand = os.path.join(eubin, "fetchcommand")
            with open(temp_fetchcommand, "w") as f:
                f.write("""
					set -e
					URI=$1
					DISTDIR=$2
					FILE=$3
					trap 'chmod a-w "${DISTDIR}"' EXIT
					chmod ug+w "${DISTDIR}"
					%s
					mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}"
				""" % orig_fetchcommand.replace("${FILE}", "${FILE}.__download__"))
            settings["FETCHCOMMAND"] = (
                f'"{BASH_BINARY}" "{temp_fetchcommand}" "${{URI}}" "${{DISTDIR}}" "${{FILE}}"'
            )
            settings.features.add("skiprocheck")
            settings.features.remove("distlocks")
            os.chmod(settings["DISTDIR"], 0o555)
            try:
                self.assertTrue(
                    bool(
                        run_async(
                            fetch, foo_uri, settings, try_mirrors=False, force=True
                        )
                    )
                )
            finally:
                settings["FETCHCOMMAND"] = orig_fetchcommand
                os.chmod(settings["DISTDIR"], orig_distdir_mode)
                settings.features.remove("skiprocheck")
                settings.features.add("distlocks")
                os.unlink(temp_fetchcommand)

            with open(foo_path, "rb") as f:
                self.assertEqual(f.read(), distfiles["foo"])

            # Test emirrordist invocation.
            emirrordist_cmd = (
                portage._python_interpreter,
                "-b",
                "-Wd",
                os.path.join(str(self.bindir), "emirrordist"),
                "--distfiles",
                settings["DISTDIR"],
                "--config-root",
                settings["EPREFIX"],
                "--delete",
                "--repositories-configuration",
                settings.repositories.config_string(),
                "--repo",
                "test_repo",
                "--mirror",
            )

            if content_db is not None:
                emirrordist_cmd = emirrordist_cmd + (
                    "--content-db",
                    content_db,
                )

            env = settings.environ()
            env["PYTHONPATH"] = ":".join(
                filter(
                    None,
                    [PORTAGE_PYM_PATH] + os.environ.get("PYTHONPATH", "").split(":"),
                )
            )

            for k in distfiles:
                try:
                    os.unlink(os.path.join(settings["DISTDIR"], k))
                except OSError:
                    pass

            proc = loop.run_until_complete(
                asyncio.create_subprocess_exec(*emirrordist_cmd, env=env)
            )
            self.assertEqual(loop.run_until_complete(proc.wait()), 0)

            for k in distfiles:
                with open(
                    os.path.join(settings["DISTDIR"], layouts[0].get_path(k)), "rb"
                ) as f:
                    self.assertEqual(f.read(), distfiles[k])

            if content_db is not None:
                loop.run_until_complete(
                    self._test_content_db(
                        emirrordist_cmd,
                        env,
                        layouts,
                        content_db,
                        distfiles,
                        settings,
                        portdb,
                    )
                )

            config_pool = config_pool_cls(settings)

            def async_fetch(pkg, ebuild_path, pre_exec=None, timeout=None):
                fetcher = EbuildFetcher(
                    config_pool=config_pool,
                    ebuild_path=ebuild_path,
                    fetchonly=False,
                    fetchall=True,
                    pkg=pkg,
                    scheduler=loop,
                    pre_exec=pre_exec,
                )
                fetcher.start()
                waiter = fetcher.async_wait()
                if timeout is not None:
                    timeout.add_done_callback(
                        lambda timeout: waiter.done() or fetcher.cancel()
                    )
                return waiter

            for cpv in ebuilds:
                metadata = dict(
                    zip(
                        Package.metadata_keys,
                        portdb.aux_get(cpv, Package.metadata_keys),
                    )
                )

                pkg = Package(
                    built=False,
                    cpv=cpv,
                    installed=False,
                    metadata=metadata,
                    root_config=root_config,
                    type_name="ebuild",
                )

                settings.setcpv(pkg)
                ebuild_path = portdb.findname(pkg.cpv)
                portage.doebuild_environment(
                    ebuild_path, "fetch", settings=settings, db=portdb
                )

                # Test good files in DISTDIR
                for k in settings["AA"].split():
                    os.stat(os.path.join(settings["DISTDIR"], k))
                self.assertEqual(
                    loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                )
                for k in settings["AA"].split():
                    with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                        self.assertEqual(f.read(), distfiles[k])

                # Test digestgen with fetch
                os.unlink(os.path.join(os.path.dirname(ebuild_path), "Manifest"))
                for k in settings["AA"].split():
                    os.unlink(os.path.join(settings["DISTDIR"], k))
                with ForkExecutor(loop=loop) as executor:
                    self.assertTrue(
                        bool(
                            loop.run_until_complete(
                                loop.run_in_executor(
                                    executor,
                                    functools.partial(
                                        digestgen, mysettings=settings, myportdb=portdb
                                    ),
                                )
                            )
                        )
                    )
                for k in settings["AA"].split():
                    with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                        self.assertEqual(f.read(), distfiles[k])

                # Test missing files in DISTDIR
                for k in settings["AA"].split():
                    os.unlink(os.path.join(settings["DISTDIR"], k))
                self.assertEqual(
                    loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                )
                for k in settings["AA"].split():
                    with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                        self.assertEqual(f.read(), distfiles[k])

                # Test interrupt, with server responses temporarily paused.
                for k in settings["AA"].split():
                    os.unlink(os.path.join(settings["DISTDIR"], k))
                loop.run_until_complete(
                    self._test_interrupt(loop, server, async_fetch, pkg, ebuild_path)
                )

                # Test empty files in DISTDIR
                for k in settings["AA"].split():
                    file_path = os.path.join(settings["DISTDIR"], k)
                    with open(file_path, "wb") as f:
                        pass
                    self.assertEqual(os.stat(file_path).st_size, 0)
                self.assertEqual(
                    loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                )
                for k in settings["AA"].split():
                    with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                        self.assertEqual(f.read(), distfiles[k])

                # Test non-empty files containing null bytes in DISTDIR
                for k in settings["AA"].split():
                    file_path = os.path.join(settings["DISTDIR"], k)
                    with open(file_path, "wb") as f:
                        f.write(len(distfiles[k]) * b"\0")
                    self.assertEqual(os.stat(file_path).st_size, len(distfiles[k]))
                self.assertEqual(
                    loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                )
                for k in settings["AA"].split():
                    with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                        self.assertEqual(f.read(), distfiles[k])

                # Test PORTAGE_RO_DISTDIRS
                settings["PORTAGE_RO_DISTDIRS"] = f'"{ro_distdir}"'
                orig_fetchcommand = settings["FETCHCOMMAND"]
                orig_resumecommand = settings["RESUMECOMMAND"]
                try:
                    settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = ""
                    for k in settings["AA"].split():
                        file_path = os.path.join(settings["DISTDIR"], k)
                        os.rename(file_path, os.path.join(ro_distdir, k))
                    self.assertEqual(
                        loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                    )
                    for k in settings["AA"].split():
                        file_path = os.path.join(settings["DISTDIR"], k)
                        self.assertTrue(os.path.islink(file_path))
                        with open(file_path, "rb") as f:
                            self.assertEqual(f.read(), distfiles[k])
                        os.unlink(file_path)
                finally:
                    settings.pop("PORTAGE_RO_DISTDIRS")
                    settings["FETCHCOMMAND"] = orig_fetchcommand
                    settings["RESUMECOMMAND"] = orig_resumecommand

                # Test local filesystem in GENTOO_MIRRORS
                orig_mirrors = settings["GENTOO_MIRRORS"]
                orig_fetchcommand = settings["FETCHCOMMAND"]
                try:
                    settings["GENTOO_MIRRORS"] = ro_distdir
                    settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = ""
                    self.assertEqual(
                        loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                    )
                    for k in settings["AA"].split():
                        with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                            self.assertEqual(f.read(), distfiles[k])
                finally:
                    settings["GENTOO_MIRRORS"] = orig_mirrors
                    settings["FETCHCOMMAND"] = orig_fetchcommand
                    settings["RESUMECOMMAND"] = orig_resumecommand

                # Test readonly DISTDIR
                orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode
                try:
                    os.chmod(settings["DISTDIR"], 0o555)
                    self.assertEqual(
                        loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                    )
                    for k in settings["AA"].split():
                        with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                            self.assertEqual(f.read(), distfiles[k])
                finally:
                    os.chmod(settings["DISTDIR"], orig_distdir_mode)

                # Test parallel-fetch mode
                settings["PORTAGE_PARALLEL_FETCHONLY"] = "1"
                try:
                    self.assertEqual(
                        loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                    )
                    for k in settings["AA"].split():
                        with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                            self.assertEqual(f.read(), distfiles[k])
                    for k in settings["AA"].split():
                        os.unlink(os.path.join(settings["DISTDIR"], k))
                    self.assertEqual(
                        loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                    )
                    for k in settings["AA"].split():
                        with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                            self.assertEqual(f.read(), distfiles[k])
                finally:
                    settings.pop("PORTAGE_PARALLEL_FETCHONLY")

                # Test RESUMECOMMAND
                orig_resume_min_size = settings["PORTAGE_FETCH_RESUME_MIN_SIZE"]
                try:
                    settings["PORTAGE_FETCH_RESUME_MIN_SIZE"] = "2"
                    for k in settings["AA"].split():
                        file_path = os.path.join(settings["DISTDIR"], k)
                        os.unlink(file_path)
                        with open(file_path + _download_suffix, "wb") as f:
                            f.write(distfiles[k][:2])
                    self.assertEqual(
                        loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                    )
                    for k in settings["AA"].split():
                        with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
                            self.assertEqual(f.read(), distfiles[k])
                finally:
                    settings["PORTAGE_FETCH_RESUME_MIN_SIZE"] = orig_resume_min_size

                # Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR
                orig_fetchcommand = settings["FETCHCOMMAND"]
                orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode
                for k in settings["AA"].split():
                    os.unlink(os.path.join(settings["DISTDIR"], k))
                try:
                    os.chmod(settings["DISTDIR"], 0o555)
                    settings["FETCHCOMMAND"] = (
                        '"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"'
                        % (BASH_BINARY, orig_fetchcommand.replace('"', '\\"'))
                    )
                    settings.features.add("skiprocheck")
                    settings.features.remove("distlocks")
                    self.assertEqual(
                        loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0
                    )
                finally:
                    settings["FETCHCOMMAND"] = orig_fetchcommand
                    os.chmod(settings["DISTDIR"], orig_distdir_mode)
                    settings.features.remove("skiprocheck")
                    settings.features.add("distlocks")

    async def _test_content_db(
        self, emirrordist_cmd, env, layouts, content_db, distfiles, settings, portdb
    ):
        # Simulate distfile digest change for ContentDB.
        emdisopts = types.SimpleNamespace(
            content_db=content_db, distfiles=settings["DISTDIR"]
        )
        with EmirrordistConfig(
            emdisopts, portdb, asyncio.get_event_loop()
        ) as emdisconf:
            # Copy revisions from bar to foo.
            for revision_key in emdisconf.content_db["filename:bar"]:
                emdisconf.content_db.add(
                    DistfileName("foo", digests=dict(revision_key))
                )

            # Copy revisions from foo to bar.
            for revision_key in emdisconf.content_db["filename:foo"]:
                emdisconf.content_db.add(
                    DistfileName("bar", digests=dict(revision_key))
                )

            content_db_state = dict(emdisconf.content_db.items())
            self.assertEqual(content_db_state, dict(emdisconf.content_db.items()))
            self.assertEqual(
                {
                    k[len("filename:") :]
                    for k in content_db_state
                    if k.startswith("filename:")
                },
                {"bar", "foo"},
            )
            self.assertEqual(
                content_db_state["filename:foo"], content_db_state["filename:bar"]
            )
            self.assertEqual(len(content_db_state["filename:foo"]), 2)

        for k in distfiles:
            try:
                os.unlink(os.path.join(settings["DISTDIR"], k))
            except OSError:
                pass

        proc = await asyncio.create_subprocess_exec(*emirrordist_cmd, env=env)
        self.assertEqual(await proc.wait(), 0)

        for k in distfiles:
            with open(
                os.path.join(settings["DISTDIR"], layouts[0].get_path(k)), "rb"
            ) as f:
                self.assertEqual(f.read(), distfiles[k])

        with EmirrordistConfig(
            emdisopts, portdb, asyncio.get_event_loop()
        ) as emdisconf:
            self.assertEqual(content_db_state, dict(emdisconf.content_db.items()))

            # Verify that remove works as expected
            filename = [filename for filename in distfiles if filename == "foo"][0]
            self.assertTrue(bool(filename.digests))
            emdisconf.content_db.remove(filename)
            # foo should still have a content revision corresponding to bar's content.
            self.assertEqual(
                {
                    k[len("filename:") :]
                    for k in emdisconf.content_db
                    if k.startswith("filename:")
                },
                {"bar", "foo"},
            )
            self.assertEqual(len(emdisconf.content_db["filename:foo"]), 1)
            self.assertEqual(
                len(
                    [
                        revision_key
                        for revision_key in emdisconf.content_db["filename:foo"]
                        if not filename.digests_equal(
                            DistfileName(
                                "foo",
                                digests=dict(revision_key),
                            )
                        )
                    ]
                ),
                1,
            )
            # bar should still have a content revision corresponding to foo's content.
            self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2)
            self.assertEqual(
                len(
                    [
                        revision_key
                        for revision_key in emdisconf.content_db["filename:bar"]
                        if filename.digests_equal(
                            DistfileName(
                                "bar",
                                digests=dict(revision_key),
                            )
                        )
                    ]
                ),
                1,
            )
            # remove the foo which refers to bar's content
            bar = [filename for filename in distfiles if filename == "bar"][0]
            foo_remaining = DistfileName("foo", digests=bar.digests)
            emdisconf.content_db.remove(foo_remaining)
            self.assertEqual(
                [
                    k[len("filename:") :]
                    for k in emdisconf.content_db
                    if k.startswith("filename:")
                ],
                ["bar"],
            )
            self.assertRaises(
                KeyError, emdisconf.content_db.__getitem__, "filename:foo"
            )
            # bar should still have a content revision corresponding to foo's content.
            self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2)

    def test_flat_layout(self):
        self.assertTrue(FlatLayout.verify_args(("flat",)))
        self.assertFalse(FlatLayout.verify_args(("flat", "extraneous-arg")))
        self.assertEqual(FlatLayout().get_path("foo-1.tar.gz"), "foo-1.tar.gz")

    def test_filename_hash_layout(self):
        self.assertFalse(FilenameHashLayout.verify_args(("filename-hash",)))
        self.assertTrue(FilenameHashLayout.verify_args(("filename-hash", "SHA1", "8")))
        self.assertFalse(
            FilenameHashLayout.verify_args(("filename-hash", "INVALID-HASH", "8"))
        )
        self.assertTrue(
            FilenameHashLayout.verify_args(("filename-hash", "SHA1", "4:8:12"))
        )
        self.assertFalse(FilenameHashLayout.verify_args(("filename-hash", "SHA1", "3")))
        self.assertFalse(
            FilenameHashLayout.verify_args(("filename-hash", "SHA1", "junk"))
        )
        self.assertFalse(
            FilenameHashLayout.verify_args(("filename-hash", "SHA1", "4:8:junk"))
        )

        self.assertEqual(
            FilenameHashLayout("SHA1", "4").get_path("foo-1.tar.gz"), "1/foo-1.tar.gz"
        )
        self.assertEqual(
            FilenameHashLayout("SHA1", "8").get_path("foo-1.tar.gz"), "19/foo-1.tar.gz"
        )
        self.assertEqual(
            FilenameHashLayout("SHA1", "8:16").get_path("foo-1.tar.gz"),
            "19/c3b6/foo-1.tar.gz",
        )
        self.assertEqual(
            FilenameHashLayout("SHA1", "8:16:24").get_path("foo-1.tar.gz"),
            "19/c3b6/37a94b/foo-1.tar.gz",
        )

    def test_content_hash_layout(self):
        self.assertFalse(ContentHashLayout.verify_args(("content-hash",)))
        self.assertTrue(ContentHashLayout.verify_args(("content-hash", "SHA1", "8")))
        self.assertFalse(
            ContentHashLayout.verify_args(("content-hash", "INVALID-HASH", "8"))
        )
        self.assertTrue(
            ContentHashLayout.verify_args(("content-hash", "SHA1", "4:8:12"))
        )
        self.assertFalse(ContentHashLayout.verify_args(("content-hash", "SHA1", "3")))
        self.assertFalse(
            ContentHashLayout.verify_args(("content-hash", "SHA1", "junk"))
        )
        self.assertFalse(
            ContentHashLayout.verify_args(("content-hash", "SHA1", "4:8:junk"))
        )

        filename = DistfileName(
            "foo-1.tar.gz",
            digests={
                algo: checksum_str(b"", hashname=algo)
                for algo in MANIFEST2_HASH_DEFAULTS
            },
        )

        # Raise KeyError for a hash algorithm SHA1 which is not in MANIFEST2_HASH_DEFAULTS.
        self.assertRaises(KeyError, ContentHashLayout("SHA1", "4").get_path, filename)

        # Raise AttributeError for a plain string argument.
        self.assertRaises(
            AttributeError, ContentHashLayout("SHA512", "4").get_path, str(filename)
        )

        self.assertEqual(
            ContentHashLayout("SHA512", "4").get_path(filename),
            "c/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
        )
        self.assertEqual(
            ContentHashLayout("SHA512", "8").get_path(filename),
            "cf/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
        )
        self.assertEqual(
            ContentHashLayout("SHA512", "8:16").get_path(filename),
            "cf/83e1/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
        )
        self.assertEqual(
            ContentHashLayout("SHA512", "8:16:24").get_path(filename),
            "cf/83e1/357eef/cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
        )

    def test_mirror_layout_config(self):
        mlc = MirrorLayoutConfig()
        self.assertEqual(mlc.serialize(), ())
        self.assertIsInstance(mlc.get_best_supported_layout(), FlatLayout)

        conf = """
[structure]
0=flat
"""
        mlc.read_from_file(io.StringIO(conf))
        self.assertEqual(mlc.serialize(), (("flat",),))
        self.assertIsInstance(mlc.get_best_supported_layout(), FlatLayout)
        self.assertEqual(
            mlc.get_best_supported_layout().get_path("foo-1.tar.gz"), "foo-1.tar.gz"
        )

        conf = """
[structure]
0=filename-hash SHA1 8:16
1=flat
"""
        mlc.read_from_file(io.StringIO(conf))
        self.assertEqual(
            mlc.serialize(), (("filename-hash", "SHA1", "8:16"), ("flat",))
        )
        self.assertIsInstance(mlc.get_best_supported_layout(), FilenameHashLayout)
        self.assertEqual(
            mlc.get_best_supported_layout().get_path("foo-1.tar.gz"),
            "19/c3b6/foo-1.tar.gz",
        )
        serialized = mlc.serialize()

        # test fallback
        conf = """
[structure]
0=filename-hash INVALID-HASH 8:16
1=filename-hash SHA1 32
2=flat
"""
        mlc.read_from_file(io.StringIO(conf))
        self.assertEqual(
            mlc.serialize(),
            (
                ("filename-hash", "INVALID-HASH", "8:16"),
                ("filename-hash", "SHA1", "32"),
                ("flat",),
            ),
        )
        self.assertIsInstance(mlc.get_best_supported_layout(), FilenameHashLayout)
        self.assertEqual(
            mlc.get_best_supported_layout().get_path("foo-1.tar.gz"),
            "19c3b637/foo-1.tar.gz",
        )

        # test deserialization
        mlc.deserialize(serialized)
        self.assertEqual(
            mlc.serialize(), (("filename-hash", "SHA1", "8:16"), ("flat",))
        )
        self.assertIsInstance(mlc.get_best_supported_layout(), FilenameHashLayout)
        self.assertEqual(
            mlc.get_best_supported_layout().get_path("foo-1.tar.gz"),
            "19/c3b6/foo-1.tar.gz",
        )

        # test erraneous input
        conf = """
[#(*DA*&*F
[structure]
0=filename-hash SHA1 32
"""
        self.assertRaises(ConfigParserError, mlc.read_from_file, io.StringIO(conf))

    def test_filename_hash_layout_get_filenames(self):
        filename = DistfileName(
            "foo-1.tar.gz",
            digests={
                algo: checksum_str(b"", hashname=algo)
                for algo in MANIFEST2_HASH_DEFAULTS
            },
        )
        layouts = (
            FlatLayout(),
            FilenameHashLayout("SHA1", "4"),
            FilenameHashLayout("SHA1", "8"),
            FilenameHashLayout("SHA1", "8:16"),
            FilenameHashLayout("SHA1", "8:16:24"),
            ContentHashLayout("SHA512", "8:8:8"),
        )

        for layout in layouts:
            distdir = tempfile.mkdtemp()
            try:
                path = os.path.join(distdir, layout.get_path(filename))
                try:
                    os.makedirs(os.path.dirname(path))
                except OSError:
                    pass

                with open(path, "wb") as f:
                    pass

                file_list = list(layout.get_filenames(distdir))
                self.assertTrue(len(file_list) > 0)
                for filename_result in file_list:
                    if isinstance(filename_result, DistfileName):
                        self.assertTrue(filename_result.digests_equal(filename))
                    else:
                        self.assertEqual(filename_result, str(filename))
            finally:
                shutil.rmtree(distdir)


# Tests only work with one ebuild at a time, so the config
# pool only needs a single config instance.
class config_pool_cls:
    def __init__(self, settings):
        self._settings = settings

    def allocate(self):
        return self._settings

    def deallocate(self, settings):
        pass
