# pylint: disable=missing-docstring
# pylint: disable=protected-access
import unittest
from unittest.mock import Mock, patch
from packaging.version import parse as parse_version

import numpy as np
from sklearn import __version__ as sklearn_version
from AnyQt.QtCore import Qt
from AnyQt.QtTest import QTest
from AnyQt.QtWidgets import QApplication
import baycomp

from Orange.classification import MajorityLearner, LogisticRegressionLearner, \
    RandomForestLearner
from Orange.classification.majority import ConstantModel
from Orange.data import Table, Domain, DiscreteVariable, ContinuousVariable
from Orange.evaluation import Results, TestOnTestData, scoring
from Orange.evaluation.scoring import ClassificationScore, RegressionScore, \
    Score
from Orange.base import Learner, Model
from Orange.modelling import ConstantLearner
from Orange.regression import MeanLearner, PLSRegressionLearner
from Orange.widgets.evaluate.owtestandscore import (
    OWTestAndScore, results_one_vs_rest)
from Orange.widgets.settings import (
    ClassValuesContextHandler, PerfectDomainContextHandler)
from Orange.widgets.tests.base import WidgetTest
from Orange.widgets.tests.utils import simulate, possible_duplicate_table
from Orange.tests import test_filename


class BadLearner(Learner):
    def fit(self, *_, **_2):  # pylint: disable=arguments-differ
        return 1 / 0


class TestOWTestAndScore(WidgetTest):
    def setUp(self):
        super().setUp()
        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore

        self.scores_domain = Domain(
            [ContinuousVariable("a"), ContinuousVariable("b")],
            [DiscreteVariable("c", values=("y", "n"))])

        self.scores_table_values = [[1, 1, 1.23, 23.8], [1., 2., 3., 4.]]

    def tearDown(self):
        self.widget.onDeleteWidget()
        super().tearDown()

    def test_basic(self):
        data = Table("iris")[::15]
        self.send_signal(self.widget.Inputs.train_data, data)
        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0)
        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertIsInstance(res, Results)
        self.assertIsNotNone(res.domain)
        self.assertIsNotNone(res.data)
        self.assertIsNotNone(res.probabilities)

        self.send_signal(self.widget.Inputs.learner, None, 0)
        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertIsNone(res)

        data = Table("housing")[::10]
        self.send_signal(self.widget.Inputs.train_data, data)
        self.send_signal(self.widget.Inputs.learner, MeanLearner(), 0)
        res = self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertIsInstance(res, Results)
        self.assertIsNotNone(res.domain)
        self.assertIsNotNone(res.data)

    def test_multiple_learners(self):
        def check_evres_names(expeced):
            res = self.get_output(self.widget.Outputs.evaluations_results)
            self.assertSequenceEqual(res.learner_names, expeced)

        data = Table("iris")[::15].copy()
        m1 = MajorityLearner()
        m1.name = "M1"
        m2 = MajorityLearner()
        m2.name = "M2"
        self.send_signal(self.widget.Inputs.train_data, data)
        self.send_signal(self.widget.Inputs.learner, m1, 1)
        self.send_signal(self.widget.Inputs.learner, m2, 2)
        res = self.get_output(self.widget.Outputs.evaluations_results)
        np.testing.assert_equal(res.probabilities[0], res.probabilities[1])
        check_evres_names(["M1", "M2"])
        self.send_signal(self.widget.Inputs.learner, None, 1)
        check_evres_names(["M2"])
        self.send_signal(self.widget.Inputs.learner, m1, 1)
        check_evres_names(["M1", "M2"])
        self.send_signal(self.widget.Inputs.learner,
                         self.widget.Inputs.learner.closing_sentinel, 1)
        check_evres_names(["M2"])
        self.send_signal(self.widget.Inputs.learner, m1, 1)
        check_evres_names(["M2", "M1"])

    def test_testOnTest(self):
        data = Table("iris")
        self.send_signal(self.widget.Inputs.train_data, data)
        self.widget.resampling = OWTestAndScore.TestOnTest
        self.send_signal(self.widget.Inputs.test_data, data)

    def test_testOnTest_incompatible_domain(self):
        iris = Table("iris")
        self.send_signal(self.widget.Inputs.train_data, iris)
        self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())
        self.widget.resampling = OWTestAndScore.TestOnTest
        # test data with the same class (otherwise the widget shows a different error)
        # and a non-nan X
        iris_test = iris.transform(
            Domain([ContinuousVariable("x")], class_vars=iris.domain.class_vars)).copy()
        with iris_test.unlocked():
            iris_test.X[:, 0] = 1
        self.send_signal(self.widget.Inputs.test_data, iris_test)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertTrue(self.widget.Error.test_data_incompatible.is_shown())

    def test_CrossValidationByFeature(self):
        data = Table("iris")
        attrs = data.domain.attributes
        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)
        data_with_disc_metas = Table.from_table(domain, data)
        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]

        self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)
        self.send_signal(self.widget.Inputs.train_data, data)
        self.assertFalse(rb.isEnabled())
        self.assertFalse(self.widget.features_combo.isEnabled())
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)

        self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)
        self.assertTrue(rb.isEnabled())
        rb.click()
        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)
        self.assertTrue(self.widget.features_combo.isEnabled())
        self.assertEqual(self.widget.features_combo.currentText(), "iris")
        self.assertEqual(len(self.widget.features_combo.model()), 1)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)

        self.send_signal(self.widget.Inputs.train_data, None)
        self.assertFalse(rb.isEnabled())
        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)
        self.assertFalse(self.widget.features_combo.isEnabled())

    def test_migrate_removes_invalid_contexts(self):
        context_invalid = ClassValuesContextHandler().new_context([0, 1, 2])
        context_valid = PerfectDomainContextHandler().new_context(*[[]] * 4)
        settings = {'context_settings': [context_invalid, context_valid]}
        self.widget.migrate_settings(settings, 2)
        self.assertEqual(settings['context_settings'], [context_valid])

    def test_migrate_shown_scores(self):
        settings = {"score_table": {"shown_scores": {"Sensitivity"}}}
        self.widget.migrate_settings(settings, 3)
        self.assertTrue(settings["score_table"]["show_score_hints"]["Sensitivity"])

    def test_memory_error(self):
        """
        Handling memory error.
        GH-2316
        """
        data = Table("iris")[::15]
        self.send_signal(self.widget.Inputs.train_data, data)
        self.assertFalse(self.widget.Error.memory_error.is_shown())

        with unittest.mock.patch(
                "Orange.evaluation.testing.Results.get_augmented_data",
                side_effect=MemoryError):
            self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=5000)
            self.assertTrue(self.widget.Error.memory_error.is_shown())

    def test_one_class_value(self):
        """
        Data with a class with one value causes widget to crash when that value
        is selected.
        GH-2351
        """
        table = Table.from_list(
            Domain(
                [ContinuousVariable("a"), ContinuousVariable("b")],
                [DiscreteVariable("c", values=("y",))]),
            list(zip(
                [42.48, 16.84, 15.23, 23.8],
                [1., 2., 3., 4.],
                "yyyy"))
        )
        self.widget.n_folds = 0
        self.assertFalse(self.widget.Error.train_data_error.is_shown())
        self.send_signal(self.widget.Inputs.train_data, table)
        self.send_signal(self.widget.Inputs.learner, MajorityLearner(), 0, wait=1000)
        self.assertTrue(self.widget.Error.train_data_error.is_shown())

    def test_data_errors(self):
        """ Test all data_errors """

        def assertErrorShown(data, is_shown, message):
            self.send_signal(self.widget.Inputs.train_data, data)
            self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())
            self.assertEqual(message, str(self.widget.Error.train_data_error))

        data = Table("iris")[::30].copy()
        with data.unlocked():
            data.Y[:] = np.nan

        iris_empty_x = Table.from_table(
            Domain([], data.domain.class_var), Table("iris")
        )

        for data, is_shown, message in zip(
                [None, data, Table("iris")[:30], iris_empty_x, data[:0]],
                [False, True, True, True, True],
                [
                    "",
                    "Target variable has no values.",
                    "Target variable has only one value.",
                    "Data has no features to learn from.",
                    "Train dataset is empty."
                ]

        ):
            assertErrorShown(data, is_shown, message)

    def test_addon_scorers(self):
        try:
            # These classes are registered, pylint: disable=unused-variable
            class NewScore(Score):
                class_types = (DiscreteVariable, ContinuousVariable)
                name = "new scorer"

                @staticmethod
                def is_compatible(domain: Domain) -> bool:
                    return True

            class NewClassificationScore(ClassificationScore):
                name = "new classification scorer"
                default_visible = False

            class NewRegressionScore(RegressionScore):
                pass

            widget = self.create_widget(OWTestAndScore)
            header = widget.score_table.view.horizontalHeader()
            self.send_signal(widget.Inputs.train_data, Table("iris"))
            scorer_names = [scorer.name for scorer in widget.scorers]
            self.assertIn("new scorer", scorer_names)
            self.assertFalse(header.isSectionHidden(3 + scorer_names.index("new scorer")))
            self.assertIn("new classification scorer", scorer_names)
            self.assertTrue(header.isSectionHidden(3 + scorer_names.index("new classification scorer")))
            self.assertNotIn("NewRegressionScore", scorer_names)
            model = widget.score_table.model


            self.send_signal(widget.Inputs.train_data, Table("housing"))
            scorer_names = [scorer.name for scorer in widget.scorers]
            self.assertIn("new scorer", scorer_names)
            self.assertNotIn("new classification scorer", scorer_names)
            self.assertIn("NewRegressionScore", scorer_names)

            self.send_signal(widget.Inputs.train_data, None)
            self.assertEqual(widget.scorers, [])
        finally:
            del Score.registry["NewScore"]  # pylint: disable=no-member
            del Score.registry["NewClassificationScore"]  # pylint: disable=no-member
            del Score.registry["NewRegressionScore"]  # pylint: disable=no-member

    def test_target_changing(self):
        data = Table("iris")
        w = self.widget  #: OWTestAndScore
        model = w.score_table.model

        w.n_folds = 2
        self.send_signal(self.widget.Inputs.train_data, data)
        self.send_signal(self.widget.Inputs.learner,
                         LogisticRegressionLearner(), 0, wait=5000)

        average_auc = float(model.item(0, 3).text())

        simulate.combobox_activate_item(w.controls.class_selection, "Iris-setosa")
        setosa_auc = float(model.item(0, 3).text())

        simulate.combobox_activate_item(w.controls.class_selection, "Iris-versicolor")
        versicolor_auc = float(model.item(0, 3).text())

        simulate.combobox_activate_item(w.controls.class_selection, "Iris-virginica")
        virginica_auc = float(model.item(0, 3).text())

        self.assertGreater(average_auc, versicolor_auc)
        self.assertGreater(average_auc, virginica_auc)
        self.assertLess(average_auc, setosa_auc)
        self.assertGreater(setosa_auc, versicolor_auc)
        self.assertGreater(setosa_auc, virginica_auc)

    def test_resort_on_data_change(self):
        iris = Table("iris")
        # one example is included from the other class
        # to keep F1 from complaining
        setosa = iris[:51]
        versicolor = iris[49:100]

        class SetosaLearner(Learner):
            def __call__(self, data):
                model = ConstantModel([1., 0, 0])
                model.domain = iris.domain
                return model

        class VersicolorLearner(Learner):
            def __call__(self, data):
                model = ConstantModel([0, 1., 0])
                model.domain = iris.domain
                return model

        # this is done manually to avoid multiple computations
        self.widget.resampling = 5
        self.widget.set_train_data(iris)
        self.widget.insert_learner(0, SetosaLearner())
        self.widget.insert_learner(1, VersicolorLearner())
        self.send_signal(self.widget.Inputs.test_data, setosa, wait=5000)

        self.widget.adjustSize()
        view = self.widget.score_table.view
        header = view.horizontalHeader()
        p = header.rect().center()
        # second visible header section (after 'Model')
        _, _, idx, *_ = (i for i in range(header.count())
                      if not header.isSectionHidden(i))
        p.setX(header.sectionPosition(idx) + 5)
        QTest.mouseClick(header.viewport(), Qt.LeftButton, pos=p)

        # Ensure that the click on header caused an ascending sort
        # Ascending sort means that wrong model should be listed first
        self.assertEqual(header.sortIndicatorOrder(), Qt.AscendingOrder)
        self.assertEqual(view.model().index(0, 0).data(), "versicolor")

        self.send_signal(self.widget.Inputs.test_data, versicolor, wait=5000)
        self.assertEqual(view.model().index(0, 0).data(), "setosa")

    def _retrieve_scores(self):
        w = self.widget
        model = w.score_table.model
        auc = model.item(0, 3).text()
        auc = float(auc) if auc != "" else None
        ca = float(model.item(0, 4).text())
        f1 = float(model.item(0, 5).text())
        precision = float(model.item(0, 6).text())
        recall = float(model.item(0, 7).text())
        return auc, ca, f1, precision, recall

    def _test_scores(self, train_data, test_data, learner, sampling, n_folds):
        w = self.widget  #: OWTestAndScore
        w.controls.resampling.buttons[sampling].click()
        if n_folds is not None:
            w.n_folds = n_folds

        self.send_signal(self.widget.Inputs.train_data, train_data)
        if test_data is not None:
            self.send_signal(self.widget.Inputs.test_data, test_data)
        self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)
        return self._retrieve_scores()

    def test_scores_constant(self):
        table = Table.from_list(
            self.scores_domain,
            list(zip(*self.scores_table_values + [list("yyyn")]))
        )

        scores = self._test_scores(
            table, table[:3], ConstantLearner(),
            OWTestAndScore.TestOnTest, None
        )
        self.assertTupleEqual(scores[1:], (1, 1, 1, 1))

        # Sklearn, we love you.
        if parse_version(sklearn_version) < parse_version("1.6dev"):
            self.assertIsNone(scores[0])
        else:
            self.assertTrue(np.isnan(scores[0]))

    def test_scores_log_reg_overfitted(self):
        table = Table.from_list(
            self.scores_domain,
            list(zip(*self.scores_table_values + [list("yyyn")]))
        )

        self.assertTupleEqual(self._test_scores(
            table, table, LogisticRegressionLearner(),
            OWTestAndScore.TestOnTest, None),
            (1, 1, 1, 1, 1))

    def test_scores_log_reg_bad(self):
        table_train = Table.from_list(
            self.scores_domain,
            list(zip(*self.scores_table_values + [list("nnny")]))
        )
        table_test = Table.from_list(
            self.scores_domain,
            list(zip(*self.scores_table_values + [list("yyyn")]))
        )

        self.assertTupleEqual(self._test_scores(
            table_train, table_test, LogisticRegressionLearner(),
            OWTestAndScore.TestOnTest, None),
            (0, 0, 0, 0, 0))

    def test_scores_log_reg_bad2(self):
        table_train = Table.from_list(
            self.scores_domain,
            list(zip(*(self.scores_table_values + [list("nnyy")]))))
        table_test = Table.from_list(
            self.scores_domain,
            list(zip(*(self.scores_table_values + [list("yynn")]))))

        lr = LogisticRegressionLearner()
        np.testing.assert_almost_equal(
            self._test_scores(
                table_train, table_test, lr, OWTestAndScore.TestOnTest, None
            ),
            (0, 0.25, 0.2, 0.1666666, 0.25),
        )

    def test_scores_log_reg_advanced(self):
        table_train = Table.from_list(
            self.scores_domain,
            list(zip([1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], "yyynn"))
        )
        table_test = Table.from_list(
            self.scores_domain,
            list(zip([1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], "yynnn"))
        )

        lr = LogisticRegressionLearner()
        np.testing.assert_
        np.testing.assert_almost_equal(
            self._test_scores(
                table_train, table_test, lr, OWTestAndScore.TestOnTest, None
            ),
            (1, 0.8, 0.8, 13 / 15, 0.8)
        )

    def test_scores_cross_validation(self):
        """
        Test more than two classes and cross-validation
        """
        self.assertTrue(
            all(x >= y for x, y in zip(
                self._test_scores(
                    Table("iris")[::15], None, LogisticRegressionLearner(),
                    OWTestAndScore.KFold, 0),
                (0.8, 0.5, 0.5, 0.5, 0.5))))

    def test_no_stratification(self):
        w = self.widget
        w.cv_stratified = True
        self._test_scores(
            Table("zoo"), None, MajorityLearner(),
            OWTestAndScore.KFold, 2)
        self.assertTrue(w.Warning.cant_stratify.is_shown())

        w.controls.cv_stratified.click()
        self.assertFalse(w.Warning.cant_stratify.is_shown())

        w.controls.cv_stratified.click()
        self.assertTrue(w.Warning.cant_stratify.is_shown())

        w.controls.n_folds.setCurrentIndex(0)
        w.controls.n_folds.activated[int].emit(0)
        self.assertFalse(w.Warning.cant_stratify.is_shown())

        w.controls.n_folds.setCurrentIndex(2)
        w.controls.n_folds.activated[int].emit(2)
        self.assertTrue(w.Warning.cant_stratify.is_shown())

        self._test_scores(
            Table("iris"), None, MajorityLearner(), OWTestAndScore.KFold, 2)
        self.assertFalse(w.Warning.cant_stratify.is_shown())

        self._test_scores(
            Table("zoo"), None, MajorityLearner(), OWTestAndScore.KFold, 2)
        self.assertTrue(w.Warning.cant_stratify.is_shown())

        self._test_scores(
            Table("housing"), None, MeanLearner(), OWTestAndScore.KFold, 2)
        self.assertFalse(w.Warning.cant_stratify.is_shown())
        self.assertTrue(w.Information.cant_stratify_numeric.is_shown())

        w.controls.cv_stratified.click()
        self.assertFalse(w.Warning.cant_stratify.is_shown())

    def test_too_many_folds(self):
        w = self.widget
        w.controls.resampling.buttons[OWTestAndScore.KFold].click()
        w.n_folds = 3
        self.send_signal(w.Inputs.train_data, Table("zoo")[:8])
        self.send_signal(w.Inputs.learner, MajorityLearner(), 0, wait=5000)
        self.assertTrue(w.Error.too_many_folds.is_shown())

    def _set_comparison_score(self, score):
        w = self.widget
        control = w.controls.comparison_criterion
        control.setCurrentText(score)
        w.comparison_criterion = control.findText(score)

    def _set_three_majorities(self):
        w = self.widget
        data = Table("iris")[::15]
        self.send_signal(w.Inputs.train_data, data)
        for i, name in enumerate(["maja", "majb", "majc"]):
            learner = MajorityLearner()
            learner.name = name
            self.send_signal(w.Inputs.learner, learner, i)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)

    @patch("baycomp.two_on_single", Mock(wraps=baycomp.two_on_single))
    def test_comparison_requires_cv(self):
        w = self.widget
        self.send_signal(w.Inputs.train_data, Table("iris")[::15])

        w.comparison_criterion = 1
        rbs = w.controls.resampling.buttons

        self._set_three_majorities()
        baycomp.two_on_single.reset_mock()

        rbs[OWTestAndScore.KFold].click()
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))
        self.assertTrue(w.compbox.isEnabled())
        baycomp.two_on_single.assert_called()
        baycomp.two_on_single.reset_mock()

        rbs[OWTestAndScore.LeaveOneOut].click()
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertIsNone(w.comparison_table.cellWidget(0, 1))
        self.assertFalse(w.compbox.isEnabled())
        baycomp.two_on_single.assert_not_called()
        baycomp.two_on_single.reset_mock()

        rbs[OWTestAndScore.KFold].click()
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))
        self.assertTrue(w.compbox.isEnabled())
        baycomp.two_on_single.assert_called()
        baycomp.two_on_single.reset_mock()

    def test_comparison_requires_multiple_models(self):
        w = self.widget
        rbs = w.controls.resampling.buttons

        self._set_three_majorities()
        w.comparison_criterion = 1

        rbs[OWTestAndScore.KFold].click()
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertTrue(w.comparison_table.isEnabled())

        self.send_signal(w.Inputs.learner, None, 1)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertTrue(w.comparison_table.isEnabled())

        self.send_signal(w.Inputs.learner, None, 2)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertFalse(w.comparison_table.isEnabled())

        rbs[OWTestAndScore.LeaveOneOut].click()
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertFalse(w.comparison_table.isEnabled())

        learner = MajorityLearner()
        learner.name = "majd"
        self.send_signal(w.Inputs.learner, learner, 1)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertFalse(w.comparison_table.isEnabled())

        rbs[OWTestAndScore.KFold].click()
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertTrue(w.comparison_table.isEnabled())

    def test_comparison_bad_slots(self):
        w = self.widget
        self._set_three_majorities()
        self._set_comparison_score("Classification accuracy")
        self.send_signal(w.Inputs.learner, BadLearner(), 2, wait=5000)
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
        self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))
        self.assertIsNone(w.comparison_table.cellWidget(0, 2))
        self.assertEqual(len(w._successful_slots()), 2)

    def test_comparison_bad_scores(self):
        w = self.widget
        self._set_three_majorities()
        self._set_comparison_score("Classification accuracy")
        self.get_output(self.widget.Outputs.evaluations_results, wait=5000)

        score_calls = -1

        def fail_on_first(*_, **_2):
            nonlocal score_calls
            score_calls += 1
            return 1 / score_calls

        with patch.object(scoring.CA, "compute_score", new=fail_on_first):
            w.update_comparison_table()

            self.assertIsNone(w.comparison_table.cellWidget(0, 1))
            self.assertIsNone(w.comparison_table.cellWidget(0, 2))
            self.assertIsNone(w.comparison_table.cellWidget(1, 0))
            self.assertIsNone(w.comparison_table.cellWidget(2, 0))
            self.assertIsNotNone(w.comparison_table.cellWidget(1, 2))
            self.assertIsNotNone(w.comparison_table.cellWidget(2, 1))
            self.assertTrue(w.Warning.scores_not_computed.is_shown())

        score_calls = -1
        with patch.object(scoring.CA, "compute_score", new=fail_on_first):
            slots = w._successful_slots()
            self.assertEqual(len(slots), 3)
            scores = w._scores_by_folds(slots)
            self.assertIsNone(scores[0])
            self.assertEqual(scores[1][0], 1)
            self.assertAlmostEqual(scores[2][0], 1 / 6)

    def test_comparison_binary_score(self):
        # false warning at call_arg.kwargs
        # pylint: disable=unpacking-non-sequence
        w = self.widget
        self._set_three_majorities()
        self._set_comparison_score("F1")
        f1mock = Mock(wraps=scoring.F1.compute_score)

        iris = Table("iris")
        with patch.object(scoring.F1, "compute_score", f1mock):
            simulate.combobox_activate_item(w.controls.class_selection,
                                            iris.domain.class_var.values[1])
            _, kwargs = f1mock.call_args
            self.assertEqual(kwargs["target"], 1)
            self.assertFalse("average" in kwargs)

            simulate.combobox_activate_item(w.controls.class_selection,
                                            iris.domain.class_var.values[2])
            _, kwargs = f1mock.call_args
            self.assertEqual(kwargs["target"], 2)
            self.assertFalse("average" in kwargs)

            simulate.combobox_activate_item(w.controls.class_selection,
                                            OWTestAndScore.TARGET_AVERAGE)
            _, kwargs = f1mock.call_args
            self.assertEqual(kwargs["average"], "weighted")
            self.assertFalse("target" in kwargs)

    def test_fill_table(self):
        w = self.widget
        self._set_three_majorities()
        scores = [object(), object(), object()]
        slots = w._successful_slots()

        def probs(p1, p2, rope):
            p1 += 1
            p2 += 1
            norm = p1 + p2 + rope * (p1 + p2)
            if rope == 0:
                return p1 / norm, p2 / norm
            else:
                return p1 / norm, rope / norm, p2 / norm

        def two_on_single(res1, res2, rope=0):
            return probs(scores.index(res1), scores.index(res2), rope)

        with patch("baycomp.two_on_single", new=two_on_single):
            for w.use_rope, w.rope in ((True, 0), (False, 0.1)):
                w._fill_table(slots, scores)
                for row in range(3):
                    for col in range(3):
                        if row == col:
                            continue
                        label = w.comparison_table.cellWidget(row, col)
                        self.assertEqual(label.text(),
                                         f"{(row + 1) / (row + col + 2):.3f}")
                        self.assertIn(f"{(row + 1) / (row + col + 2):.3f}",
                                      label.toolTip())

            w.use_rope = True
            w.rope = 0.25
            w._fill_table(slots, scores)
            for row in range(3):
                for col in range(3):
                    if row == col:
                        continue
                    label = w.comparison_table.cellWidget(row, col)
                    for text in (label.text(), label.toolTip()):
                        self.assertIn(f"{probs(row, col, w.rope)[0]:.3f}", text)
                        self.assertIn(f"{probs(row, col, w.rope)[1]:.3f}", text)

    def test_nan_on_comparison(self):
        w = self.widget
        w.use_rope = True
        self._set_three_majorities()
        scores = [object(), object(), object()]
        slots = w._successful_slots()

        def two_on_single(_1, _2, rope=0):
            if rope:
                return np.nan, np.nan, np.nan
            else:
                return np.nan, np.nan

        with patch("baycomp.two_on_single", new=two_on_single):
            for w.rope in (0, 0.1):
                w._fill_table(slots, scores)
                label = w.comparison_table.cellWidget(1, 0)
                self.assertEqual(label.text(), "NA")

    def test_unique_output_domain(self):
        data = possible_duplicate_table('random forest')
        self.send_signal(self.widget.Inputs.train_data, data)
        self.send_signal(self.widget.Inputs.learner, RandomForestLearner(), 0)
        output = self.get_output(self.widget.Outputs.predictions)
        self.assertEqual(output.domain.metas[0].name, 'random forest (1)')

    def test_copy_to_clipboard(self):
        self.send_signal(self.widget.Inputs.train_data, Table("iris"))
        self.send_signal(self.widget.Inputs.learner, RandomForestLearner(), 0)
        self.wait_until_finished()
        view = self.widget.score_table.view
        model = self.widget.score_table.model
        selection_model = view.selectionModel()
        selection_model.select(model.index(0, 0),
                               selection_model.Select | selection_model.Rows)
        self.widget.copy_to_clipboard()
        clipboard_text = QApplication.clipboard().text()
        # Tests appear to register additional scorers, so we clip the list
        # to what we know to be there and visible
        clipboard_text = "\t".join(clipboard_text.split("\t")[:6]).strip()
        view_text = "\t".join([str(model.data(model.index(0, i)))
                               for i in (0, 3, 4, 5, 6, 7)]).strip()
        self.assertEqual(clipboard_text, view_text)

    def test_multi_target_input(self):
        class NewScorer(Score):
            class_types = (
                ContinuousVariable,
                DiscreteVariable,
            )

            @staticmethod
            def is_compatible(domain: Domain) -> bool:
                return True

            def compute_score(self, results):
                return [0.75]

        domain = Domain([ContinuousVariable('var1')],
                        class_vars=[
                            ContinuousVariable('c1'),
                            DiscreteVariable('c2', values=('no', 'yes'))
                        ])
        data = Table.from_list(domain, [[1, 5, 0], [2, 10, 1], [2, 10, 1]])

        mock_model = Mock(spec=Model, return_value=np.asarray([0.2, 0.1, 0.2]))
        mock_model.name = 'Mockery'
        mock_model.domain = domain
        mock_learner = Mock(spec=Learner, return_value=mock_model)
        mock_learner.name = 'Mockery'

        widget = self.create_widget(OWTestAndScore)
        widget.resampling = OWTestAndScore.TestOnTrain
        self.send_signal(widget.Inputs.train_data, data)
        self.send_signal(widget.Inputs.learner, MajorityLearner(), 0)
        self.send_signal(widget.Inputs.learner, mock_learner, 1)
        _ = self.get_output(widget.Outputs.evaluations_results, wait=5000)
        self.assertTrue(len(widget.scorers) == 1)
        self.assertTrue(NewScorer in widget.scorers)
        self.assertTrue(len(widget._successful_slots()) == 1)

    def test_multiple_targets_pls(self):
        housing = Table("housing")
        class_vars = [housing.domain.class_var, housing.domain.attributes[0]]
        domain = Domain(housing.domain.attributes[1:], class_vars=class_vars)
        multiple_targets_data = housing.transform(domain)

        self.widget.error = Mock()
        self.send_signal(self.widget.Inputs.train_data, multiple_targets_data)
        self.send_signal(self.widget.Inputs.learner, PLSRegressionLearner())
        self.wait_until_finished()
        self.assertIn("Multiple targets are not supported.",
                      self.widget.error.call_args[0][0])


class TestHelpers(unittest.TestCase):
    def test_results_one_vs_rest(self):
        data = Table(test_filename("datasets/lenses.tab"))
        learners = [MajorityLearner()]
        res = TestOnTestData()(data[1::2], data[::2], learners=learners)
        r1 = results_one_vs_rest(res, pos_index=0)
        r2 = results_one_vs_rest(res, pos_index=1)
        r3 = results_one_vs_rest(res, pos_index=2)

        np.testing.assert_almost_equal(np.sum(r1.probabilities, axis=2), 1.0)
        np.testing.assert_almost_equal(np.sum(r2.probabilities, axis=2), 1.0)
        np.testing.assert_almost_equal(np.sum(r3.probabilities, axis=2), 1.0)

        np.testing.assert_almost_equal(
            r1.probabilities[:, :, 1] +
            r2.probabilities[:, :, 1] +
            r3.probabilities[:, :, 1],
            1.0
        )
        self.assertEqual(r1.folds, res.folds)
        self.assertEqual(r2.folds, res.folds)
        self.assertEqual(r3.folds, res.folds)

        np.testing.assert_equal(r1.row_indices, res.row_indices)
        np.testing.assert_equal(r2.row_indices, res.row_indices)
        np.testing.assert_equal(r3.row_indices, res.row_indices)


if __name__ == "__main__":
    unittest.main()
