%% This Source Code Form is subject to the terms of the Mozilla Public
%% License, v. 2.0. If a copy of the MPL was not distributed with this
%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
%%
%% Copyright (c) 2007-2023 VMware, Inc. or its affiliates.  All rights reserved.
%%
-module(mqtt_node).

-export([start/0, node_id/0, server_id/0, all_node_ids/0, leave/1, trigger_election/0,
         delete/1]).

-define(ID_NAME, mqtt_node).
-define(START_TIMEOUT, 100_000).
-define(RETRY_INTERVAL, 5000).
-define(RA_OPERATION_TIMEOUT, 60_000).
-define(RA_SYSTEM, coordination).

node_id() ->
    server_id(node()).

server_id() ->
    server_id(node()).

server_id(Node) ->
    {?ID_NAME, Node}.

all_node_ids() ->
    [server_id(N) || N <- rabbit_nodes:list_members(),
                   can_participate_in_clientid_tracking(N)].

start() ->
    %% 3s to 6s randomized
    Repetitions = rand:uniform(10) + 10,
    start(300, Repetitions).

start(_Delay, AttemptsLeft) when AttemptsLeft =< 0 ->
    ok = start_server(),
    trigger_election();
start(Delay, AttemptsLeft) ->
    NodeId = server_id(),
    Nodes = compatible_peer_servers(),
    case ra_directory:uid_of(?RA_SYSTEM, ?ID_NAME) of
          undefined ->
              case Nodes of
                  [] ->
                      %% Since cluster members are not known ahead of time and initial boot can be happening in parallel,
                      %% we wait and check a few times (up to a few seconds) to see if we can discover any peers to
                      %% join before forming a cluster. This reduces the probability of N independent clusters being
                      %% formed in the common scenario of N nodes booting in parallel e.g. because they were started
                      %% at the same time by a deployment tool.
                      %%
                      %% This scenario does not guarantee single cluster formation but without knowing the list of members
                      %% ahead of time, this is a best effort workaround. Multi-node consensus is apparently hard
                      %% to achieve without having consensus around expected cluster members.
                      rabbit_log:info("MQTT: will wait for ~tp more ms for cluster members to join before triggering a Raft leader election", [Delay]),
                      timer:sleep(Delay),
                      start(Delay, AttemptsLeft - 1);
                  Peers ->
                      %% Trigger an election.
                      %% This is required when we start a node for the first time.
                      %% Using default timeout because it supposed to reply fast.
                      rabbit_log:info("MQTT: discovered cluster peers that support client ID tracking: ~p", [Peers]),
                      ok = start_server(),
                      _ = join_peers(NodeId, Peers),
                      ra:trigger_election(NodeId, ?RA_OPERATION_TIMEOUT)
              end;
          _ ->
              _ = join_peers(NodeId, Nodes),
              ok = ra:restart_server(?RA_SYSTEM, NodeId),
              ra:trigger_election(NodeId, ?RA_OPERATION_TIMEOUT)
    end.

compatible_peer_servers() ->
    all_node_ids() -- [(node_id())].

start_server() ->
    NodeId = node_id(),
    Nodes = compatible_peer_servers(),
    UId = ra:new_uid(ra_lib:to_binary(?ID_NAME)),
    Timeout = application:get_env(kernel, net_ticktime, 60) + 5,
    Conf = #{cluster_name => ?ID_NAME,
             id => NodeId,
             uid => UId,
             friendly_name => atom_to_list(?ID_NAME),
             initial_members => Nodes,
             log_init_args => #{uid => UId},
             tick_timeout => Timeout,
             machine => {module, mqtt_machine, #{}}
            },
    rabbit_log:info("MQTT: starting Ra server with initial members: ~p", [Nodes]),
    ra:start_server(?RA_SYSTEM, Conf).

trigger_election() ->
    ra:trigger_election(server_id(), ?RA_OPERATION_TIMEOUT).

join_peers(_NodeId, []) ->
    ok;
join_peers(NodeId, Nodes) ->
    join_peers(NodeId, Nodes, 100).

join_peers(_NodeId, _Nodes, RetriesLeft) when RetriesLeft =:= 0 ->
    rabbit_log:error("MQTT: exhausted all attempts while trying to rejoin cluster peers");
join_peers(NodeId, Nodes, RetriesLeft) ->
    case ra:members(Nodes, ?START_TIMEOUT) of
        {ok, Members, _} ->
            case lists:member(NodeId, Members) of
                true  -> ok;
                false -> ra:add_member(Members, NodeId)
            end;
        {timeout, _} ->
            rabbit_log:debug("MQTT: timed out contacting cluster peers, %s retries left", [RetriesLeft]),
            timer:sleep(?RETRY_INTERVAL),
            join_peers(NodeId, Nodes, RetriesLeft - 1);
        Err ->
            Err
    end.

-spec leave(node()) -> 'ok' | 'timeout' | 'nodedown'.
leave(Node) ->
    NodeId = server_id(),
    ToLeave = server_id(Node),
    try
        ra:leave_and_delete_server(?RA_SYSTEM, NodeId, ToLeave)
    catch
        exit:{{nodedown, Node}, _} ->
            nodedown
    end.

can_participate_in_clientid_tracking(Node) ->
    case rpc:call(Node, mqtt_machine, module_info, []) of
        {badrpc, _} -> false;
        _           -> true
    end.

-spec delete(Args) -> Ret when
      Args :: rabbit_feature_flags:enable_callback_args(),
      Ret :: rabbit_feature_flags:enable_callback_ret().
delete(_) ->
    RaNodes = all_node_ids(),
    Nodes = lists:map(fun({_, N}) -> N end, RaNodes),
    LockId = {?ID_NAME, node_id()},
    rabbit_log:info("Trying to acquire lock ~p on nodes ~p ...", [LockId, Nodes]),
    true = global:set_lock(LockId, Nodes),
    rabbit_log:info("Acquired lock ~p", [LockId]),
    try whereis(?ID_NAME) of
        undefined ->
            rabbit_log:info("Local Ra process ~s does not exist", [?ID_NAME]),
            ok;
        _ ->
            rabbit_log:info("Deleting Ra cluster ~s ...", [?ID_NAME]),
            try ra:delete_cluster(RaNodes, 15_000) of
                {ok, _Leader} ->
                    rabbit_log:info("Successfully deleted Ra cluster ~s", [?ID_NAME]),
                    ok;
                {error, Reason} ->
                    rabbit_log:info("Failed to delete Ra cluster ~s: ~p", [?ID_NAME, Reason]),
                    ServerId = server_id(),
                    case ra:force_delete_server(?RA_SYSTEM, ServerId) of
                        ok ->
                            rabbit_log:info("Successfully force deleted Ra server ~p", [ServerId]),
                            ok;
                        Error ->
                            rabbit_log:error("Failed to force delete Ra server ~p: ~p",
                                             [ServerId, Error]),
                            {error, Error}
                    end
            catch exit:{{shutdown, delete}, _StackTrace} ->
                      rabbit_log:info("Ra cluster ~s already being deleted", [?ID_NAME]),
                      ok
            end
    after
        true = global:del_lock(LockId, Nodes),
        rabbit_log:info("Released lock ~p", [LockId])
    end.
