%%% -----------------------------------------------------------------------------
%%% File    : inviso_tool.erl
%%% Author  : Lennart Öhman <lennart.ohman@st.se>
%%% Description : 
%%%
%%% Created : 22 Sep 2005 by Lennart Öhman <lennart.ohman@st.se>
%%% This is the tool that uses the inviso trace functionality.
%%% It replaces ttb in many aspects.
%%% -----------------------------------------------------------------------------
-module(inviso_tool).

%% ------------------------------------------------------------------------------
%% API exports.
%% ------------------------------------------------------------------------------
-export([start/3,start/2,start/1,start/0]).
-export([nodes/0]).
-export([start_session/1,cancel_session/1,stop_session/3]).


%% ------------------------------------------------------------------------------
%% Internal exports.
%% ------------------------------------------------------------------------------
-export([init/1,handle_call/3,handle_info/2]).



%% ------------------------------------------------------------------------------
%% Module-wide constants.
%% ------------------------------------------------------------------------------

-define(SERVER,inviso_tool).                 % The tool control process name.
-define(DEFAULT_TIMEOUT,10000).              % Gen-server server side timeout.
-define(LOCAL_RUNTIME,local_runtime).        % Name for node when non-distributed.


%% A standard mandatory safety-catch.
-define(SC,[]).
%% ------------------------------------------------------------------------------

%% Constants used as options to the trace control component.
%% TraceControlComponentOptions.
-define(TCCO_SUBSCRIBE,{subscribe,self()}).
%% ------------------------------------------------------------------------------

%% Constants describing the events which may arrive from our trace control component.
%% TraceControlComponentEvents
-define(TCCE_DISCONNECTED,left_cluster).
-define(TCCE_CONNECTED,foobar).              % TBD!
%% ------------------------------------------------------------------------------

%% Constants for events generated by inviso_tool to our subscribers.
-define(E_DISCONNECT_NODE,disconnect_node).
-define(E_CONNECT_NODE,connect_node).
-define(E_END_SESSION,session_ended).
%% ------------------------------------------------------------------------------



%% ==============================================================================
%% Exported API functions.
%% ==============================================================================


%% ------------------------------------------------------------------------------
%% start/3,/2,/1,/0 = {ok,Pid}.
%%   CtrlNode=atom( )Nodename where the tracer control component shall run.
%%   TracerNodes=list() Nodes where tracing shall be conducted.
%%   SafetyCatches=[{Mod,Func},...]
%%
%% Start function which spawns the trace-tool-builder control process. This function
%% comes in two major versions, for a distributed system and for a non-distributed
%% system. When non-distributed all components of the trace system are supposed to
%% run on this node. Non-distributed functionality of underlaying tracing will be
%% used.
start(CtrlNode,TraceNodes,Options) ->
    gen_server:start({local,?SERVER},
		     ?MODULE,
		     {{node,CtrlNode},TraceNodes,Options},
		     []).
start(CtrlNode,TraceNodes) ->
    gen_server:start({local,?SERVER},?MODULE,{{node,CtrlNode},TraceNodes,[]},[]).
start(Options) ->
    gen_server:start({local,?SERVER},?MODULE,{void,void,Options},[]).
start() ->
    gen_server:start({local,?SERVER},?MODULE,{void,void,[]},[]).
%% ------------------------------------------------------------------------------

%% nodes/0 = Nodes
%%   Nodes=[{NodeName,Status},...].
%%   Status=running | unavailable
%%
%% Returns a list of all nodes we have asked to be part of our set. Meaning having
%% a runtime component controlled by "our" control component.
nodes() ->
    gen_server:call(?SERVER,nodes).
%% ------------------------------------------------------------------------------


% run(InitMod,SuiteMod,SuiteArgs) ->
%     gen_server:call(?SERVER,{run,InitMod,SuiteMod,SuiteArgs}).

%% ==============================================================================
%% Exported trace functions.
%% ==============================================================================

%% start_session(NodeParams) = {ok,{SID,Result}} | {error,Reason}
%% start_session(TracerData) =
%%   NodeParams=[{Node,TracerData},...]
%%   SID=pid(), tool session identifier.
%%   Result=[{Node,NodeResult},...] | NodeResult
%%   NodeResult= TBD!!!
%%
%% Function starting a session managed by the control-process. The nodes mentioned in
%% the NodeParams list must previously have been specified as our nodes. In the
%% non-distributed case, just provide tracer-data as argument.
start_session(NodeParams) when list(NodeParams) ->
    gen_server:call(?SERVER,{start_session,NodeParams});
start_session(TracerData) when tuple(TracerData); atom(TracerData) ->
    gen_server:call(?SERVER,{start_session_non_distributed,TracerData}).
%% ------------------------------------------------------------------------------

%% stop_session(SID) = {ok,Nodes} | ok | {error,Reason}
%%   Nodes=[NodeName,...]
%%   Reason=unknown_session
%%
%% Function which stops an ongoing session. This means loosing all information
%% related to this session. Lookin in the session handler for further info on
%% what happens when a session is stopped.
cancel_session(SID) when pid(SID) ->
    gen_server:call(?SERVER,{cancel_session,SID}).
%% ------------------------------------------------------------------------------


stop_session(SID,Dir,Prefix) when pid(SID) ->
    gen_server:call(?SERVER,{stop_session,SID,Dir,Prefix}).


%% run_suite(Params) -> {ok,SuiteRef,Result} | {error,Reason}
%%   SuiteRef=A reference to the suite.
%%   Result=[{SessionName,SessionReturn},...]
%%   SessionReturn=ok|{error,Reason}
%% Starts a suite, that is one or several sessions. The suite ends when all
%% sessions have finished and made to to end by the suite. Note that ok will
%% be returned for a session as long as no general failure occured. This means
%% that even if no nodes at all were initiated, it may be ok.
% run_suite(Params) ->
%     gen_server:call(?SERVER,{run_suite,Params}).



%% ==============================================================================
%% Genserver call-backs.
%% ==============================================================================

%% The init function for the trace tool builder control process.
%% It will start the trace control component and run-time components on all
%% Erlang nodes participating in the tracing.
init({CtrlNode,TraceNodes,Options}) ->
    process_flag(trap_exit,true),
    case get_and_check_init_options(Options) of
	{ok,SafetyCa,TimeOut,EventPid,Dbg,RTtag} ->
	    case CtrlNode of
		void ->                      % Non distributed case.
		    case start_trace_control(Dbg,RTtag) of
			{ok,{Pid,Info}} ->
			    {ok,
			     mk_ld(CtrlNode,Pid,[{local_runtime,Info}],?SC++SafetyCa,
				   TimeOut,EventPid,Dbg,RTtag),
			     TimeOut};
			{error,Reason} ->    % Could not start the tracer!
			    {stop,Reason}
		    end;
		{node,NodeName} ->
		    case start_trace_control(NodeName,TraceNodes,Dbg,RTtag) of
			{ok,{Pid,Result}} ->
			    {ok,
			     mk_ld(CtrlNode,Pid,Result,?SC++SafetyCa,TimeOut,EventPid,Dbg,RTtag),
			     TimeOut};
			{error,Reason} ->    % Could not start the tracer!
			    {stop,Reason}
		    end;
		{M,F,Args} when atom(M),atom(F),list(Args) ->
		    case apply(M,F,Args) of
			NodeName when atom(NodeName) ->
			    case start_trace_control(NodeName,TraceNodes,Dbg,RTtag) of
				{ok,{Pid,Result}} ->
				    {ok,
				     mk_ld({CtrlNode,NodeName},Pid,Result,?SC++SafetyCa,
					   TimeOut,EventPid,Dbg,RTtag),
				     TimeOut};
				{error,Reason} ->
				    {stop,Reason}
			    end;
			Else ->
			    {stop,{faulty_nodename,Else}}
		    end
	    end;
	{error,What} ->
	    {stop,{faulty_options,What}}
    end.
%% -----------------------------------------------------------------------------


%% Help function which starts the tracer-control-component on the
%% desired Erlang node. It also tells the control component where to start or
%% possibly adopt run-time components.
%% This function also makes the inviso tool control process subscribe to event from
%% the trace control component.
%% Returns {ok,{TracerControlPid,Result}} or {error,Reason}, where
%% Result is [{Node,Info},...] or Info (the later in the non distributed case).
%% Info is {ok,new}, {ok,{tag,Tag}} or {error,Reason}.
start_trace_control(Dbg,RTtag) ->
    case inviso:start([?TCCO_SUBSCRIBE]) of % This is the non distributed case.
	{ok,Pid} ->
	    case inviso:add_node(trace_c_tag(RTtag)) of
		{ok,{adopted,State,Status,Tag}} when Tag=/=RTtag -> % Other tag, stop it!
		    case inviso:stop_node() of
			ok ->
			    inviso_tool_lib:debug(stop_wrong_tag,
						  Dbg,
						  [{ok,{adopted,State,Status,Tag}},ok]),
			    {ok,{Pid,{error,wrong_tag}}};
			{error,Reason} ->    % Extremely strange!
			    inviso:stop(),
			    {error,{stop_node,Reason}}
		    end;
		{ok,Result} ->               % All other cases of ok are ok!
		    {ok,{Pid,{ok,Result}}};
		{error,already_added} ->     % Maybe it autoconnected?!
		    {ok,{Pid,{error,already_added}}}; % Wait for the connect event.
		{error,Reason} ->            % Failed to start a local rt component.
		    inviso:stop(),          % Stop trace_c!
		    {error,{local_runtime,Reason}}
	    end;
	{error,Reason} ->
	    {error,{start_trace_c,Reason}}
    end.
		    
start_trace_control(CtrlNode,TraceNodes,Dbg,RTtag) ->
    case inviso_tool_lib:inviso_cmd(CtrlNode,start,[[?TCCO_SUBSCRIBE]]) of
	{ok,Pid} ->
	    case inviso_tool_lib:inviso_cmd(CtrlNode,add_nodes,[TraceNodes,trace_c_tag(RTtag)]) of
		{ok,Result} ->               % May contain some errors too.
		    {ok,Result2}=stop_adopted_nodes(CtrlNode,RTtag,Result,Dbg),
		    {ok,{Pid,Result2}};
		{error,Reason} ->            % Strange, started trace_c but failed here.
		    inviso_tool_lib:inviso_cmd(CtrlNode,stop_all,[]),
		    {error,{add_nodes,Reason}}
	    end;
	{error,Reason} ->                    % Maybe already started.
	    {error,{start_trace_c,Reason}}
    end.
%% -----------------------------------------------------------------------------
	    



% handle_call({run,InitMod,TraceCases,StopCond},_From,LD) ->
%     P=spawn_link(?MODULE,trace_case_handler,[TraceCases]),

handle_call(nodes,_From,LD) ->
    NodeData=handle_nodes(get_nodedata_ld(LD)),
    {reply,NodeData,LD,get_timeout_ld(LD)};

handle_call({start_session_non_distributed,TracerData},From,LD) ->
    handle_call({start_session,[{?LOCAL_RUNTIME,TracerData}]},From,LD);

handle_call({start_session,NodeParams},From,LD) ->
    CtrlNode=get_ctrlnode_node_ld(LD),
    CtrlPid=get_ctrlpid_ld(LD),
    SC=get_catches_ld(LD),
    Dbg=get_dbg_ld(LD),
    {NodesIn,NodesNotIn}=not_in_session(ld_nodedata_mk_list(get_nodedata_ld(LD)),NodeParams),
    case
	case CtrlNode of
	    void ->                          % Use non-distributed functions.
		{_,TracerData}=NodeParams,
		inviso_tool_sh:start_link(From,TracerData,CtrlPid,SC,Dbg);
	    _ ->
		inviso_tool_sh:start_link(From,NodeParams,CtrlNode,CtrlPid,SC,
					  Dbg,NodesIn,NodesNotIn)
	end of
	{ok,SID} ->                          % The pid is the SID.
	    NewLD=add_session_ld(NodesNotIn,SID,LD), % Add the new session to loopdata.
	    {noreply,NewLD,get_timeout_ld(NewLD)}; % Reply will come from SID-pid.
	{error,_Reason} ->
	    {noreply,LD,get_timeout_ld(LD)}
    end;

handle_call({cancel_session,SID},_From,LD) ->
    Sessions=get_sessions_ld(LD),
    case lists:member(SID,Sessions) of       % Check amongst our SIDs.
	true ->                              % Ok, it is a valid SID!
	    inviso_tool_sh:cancel_session(SID),
	    case delete_session_ld(SID,LD) of
		{NewLD,?LOCAL_RUNTIME} ->    % The non-distributed case.
		    {reply,ok,NewLD,get_timeout_ld(NewLD)};
		{NewLD,Nodes} ->
		    {reply,{ok,Nodes},NewLD,get_timeout_ld(NewLD)}
	    end;
	false ->                             % Incorrect SID.
	    {reply,{error,unknown_session},LD,get_timeout_ld(LD)}
    end;

handle_call({stop_session,SID,Dir,Prefix},_From,LD) ->
    Sessions=get_sessions_ld(LD),
    case lists:member(SID,Sessions) of       % Check amongst our SIDs.
	true ->                              % Ok, it is a valid SID!
	    case inviso_tool_sh:stop_session(SID,Dir,Prefix) of
		{ok,{FailedNodes,FetchedFiles}} ->
		    case delete_session_ld(SID,LD) of
			{NewLD,[?LOCAL_RUNTIME]} ->  % The non-distributed case.
			    {reply,{ok,[{failed_nodes,FailedNodes},
					{files,FetchedFiles}]},
			     NewLD,
			     get_timeout_ld(NewLD)};
			{NewLD,Nodes} ->     % Nodes we removed from the session.
			    {reply,{ok,[{nodes,Nodes},
					{failed_nodes,FailedNodes},
					{files,FetchedFiles}]},
			     NewLD,
			     get_timeout_ld(NewLD)}
		    end
	    end;
	false ->                             % Incorrect SID.
	    {reply,{error,unknown_session},LD,get_timeout_ld(LD)}
    end;

handle_call(What,_From,LD) ->
    inviso_tool_lib:debug(garbage,get_dbg_ld(LD),[What]),
    {reply,{error,unknown_request},LD}.

% handle_call({run_suite,Sessions},_From,LD) ->
%     {SuiteID,SData,Result}=handle_run_suite(Sessions,LD),
%     {reply,{ok,SuiteID,Result},update_sdata_ld(SData,LD)};
% handle_call() ->


%% Handling of periodic timeouts if used.
%% We search for new nodes, that is runtime components listed as ours but
%% not been able to own yet.
handle_info(timeout,LD) ->
    case timeout_check_new_nodes(get_ctrlnode_node_ld(LD),
				 get_nodedata_ld(LD),
				 get_rttag_ld(LD),
				 get_dbg_ld(LD)) of
	false ->                             % No new nodes found, or all connected.
	    {noreply,LD,get_timeout_ld(LD)};
	{ok,NewNodeData} ->                  % New nodes found and connected.
	    handle_send_event(get_eventpid_ld(LD),?E_CONNECT_NODE),
	    {noreply,put_nodedata_ld(NewNodeData,LD),get_timeout_ld(LD)};
	{error,_Reason} ->                   % Some searious problem occured.
	    {noreply,LD,get_timeout_ld(LD)}  % Here we can actually become inconsistent!
    end;

%% Receiving subscribed events from the trace control component.
handle_info({trace_event,CtrlPid,_When,{disconnected,Node,_}},LD) ->
    case get_ctrlpid_ld(LD) of
	CtrlPid ->                           % It is indeed from our control component.
	    case trace_event_disconnect_node(Node,get_nodedata_ld(LD)) of
		{ok,NewNodeData} ->          % It was one of our nodes.
		    handle_send_event(get_eventpid_ld(LD),?E_DISCONNECT_NODE),
		    inviso_tool_lib:debug(trace_event,get_dbg_ld(LD),[?TCCE_DISCONNECTED,Node]),
		    {noreply,put_nodedata_ld(NewNodeData,LD),get_timeout_ld(LD)};
		false ->                     % It was not one of our nodes.
		    {noreply,LD,get_timeout_ld(LD)}  % Ignore it!
	    end;
	_ ->                                 % Hmm, not from our control component!
	    {noreply,LD,get_timeout_ld(LD)}  % Ignore it!
    end;

%% A runtime component connects to our trace control component. Either because
%% we ordered it to, or because it spontaneously wanted to. We do only accept
%% runtime components having our reference tag and that are specified as being
%% one of our nodes.
handle_info({trace_event,CtrlPid,_When,{connected,Node,{Tag,_}}},LD) ->
    case get_ctrlpid_ld(LD) of
	CtrlPid ->                           % It is indeed from our control component.
	    case trace_event_is_my_tag(Tag,get_rttag_ld(LD)) of
		true ->                      % This node is allowed join.
		    case trace_event_connect_node(Node,get_nodedata_ld(LD)) of
			{ok,NewNodeData} ->
			    handle_send_event(get_eventpid_ld(LD),?E_CONNECT_NODE),
			    inviso_tool_lib:debug(trace_event,
						  get_dbg_ld(LD),
						  [?TCCE_CONNECTED,Node]),
			    {noreply,put_nodedata_ld(NewNodeData,LD),get_timeout_ld(LD)};
			false ->             % The node is not set-up as our node.
			    {noreply,LD,get_timeout_ld(LD)}
		    end;
		false ->                     % Not previously started by "us".
		    inviso_tool_lib:inviso_cmd(get_ctrlnode_node_ld(LD),stop_nodes,[[Node]]),
		    inviso_tool_lib:debug(trace_event,get_dbg_ld(LD),[?TCCE_CONNECTED,Node]),
		    inviso_tool_lib:debug(trace_event,get_dbg_ld(LD),[stopped_node,Node]),
		    {noreply,LD,get_timeout_ld(LD)}
	    end;
	_ ->                                 % Hmm, not from our control component!
	    {noreply,LD,get_timeout_ld(LD)}  % Ignore it!
    end;

%% We expect exit messages from our session handlers. If a session handler has
%% terminated we want to clean up and mark participating nodes as no longer part
%% of the no longer existing session.
%% We distinguage between the 'normal' reason and other terminations. If it
%% terminates normally, the session handler has been stopped due to that we are
%% actually stopping the session. No cleaning necessary then (since the handle
%% function for stop cleans!).
handle_info({'EXIT',_,normal},LD) ->         % Do nothing when reason 'normal'.
    {noreply,LD,get_timeout_ld(LD)};         % The SID is already removed!(?).
handle_info({'EXIT',Pid,Reason},LD) ->
    case lists:member(Pid,get_sessions_ld(LD)) of % Is it a SID?
	true ->                              % It was one of our sessions.
	    {NewNodeData,Nodes}=ld_nodedata_remove_sid(Pid,get_nodedata_ld(LD)),
	    io:format("Session handler ~w crashed:~n~p~n",[Pid,Reason]),
	    handle_send_event(get_eventpid_ld(LD),[?E_END_SESSION,Pid,Nodes]),
	    inviso_tool_lib:debug(session_handler,get_dbg_ld(LD),[Pid,Reason,Nodes]),
	    {noreply,put_nodedata_ld(NewNodeData,LD),get_timeout_ld(LD)};
	false ->                             % It was not one of our sessions.
	    {noreply,LD,get_timeout_ld(LD)}
    end;

handle_info(_,LD) ->
    {noreply,LD,get_timeout_ld(LD)}.


%% ==============================================================================
%% First level help functions to handle_call functions.
%% ==============================================================================

%% Help function to nodes/0 call-back. Nodes is the list from our loop data
%% containing the status of all nodes, connected and not connected.
handle_nodes(NodeData) ->
    handle_nodes_2(ld_nodedata_mk_list(NodeData)).

handle_nodes_2([{Node,Status,_SID}|Rest]) ->
    [{Node,ld_nodedata_translate_to_ui_indication(Status)}|handle_nodes_2(Rest)];
handle_nodes_2([]) ->
    [].
%% ------------------------------------------------------------------------------



%% ------------------------------------------------------------------------------


%% ------------------------------------------------------------------------------
%% Help functions for run_suite.
%% ------------------------------------------------------------------------------

%% Help function which starts the sessions of this suite. A session handling
%% process is spawned for each session successfully started with the trace control
%% component.
%% Returns {SuiteID,SessionDataStructur,Result}.
% handle_run_suite(Sessions,LD) ->
%     SData=lookup_sdata_ld(LD),
%     CtrlNode=lookup_ctrlnode_ld(LD),
%     handle_run_suite_2(Sessions,CtrlNode,SData,make_ref(),[]).


% handle_run_suite_2([{SName,Session}|Rest],CtrlNode,SData,SuiteID,Accum) ->
%     case h_run_suite_start_session(CtrlnNode,Session) of
% 	{ok,SessionID,Result,FileInstr} ->   % Started a session..
% 	    Pid=session_handler(CtrlNode,SessionID,Result,TraceCases,FileInstr),
% 	    NewSData=put_session_sdata(SName,SessionID,SuiteID,Pid),
% 	    handle_run_suite_2(Rest,CtrlNode,NewSData,[{SName,ok}|Accum]);
% 	{error,Reason} ->                    % Could not start the session.
% 	    NewSData=put_session_sdata(SName,error,SuiteID,void),
% 	    handle_run_suite_2(Rest,CtrlNode,NewSData,SuiteID,[{SName,{error,Reason}}|Accum])
%     end;
% handle_run_suite_2([],_,SData,SuiteID,Accum) ->
%     {SuiteID,SData,Accum}.

% h_run_suite_start_session(void,Session) ->   % Non distributed case.
%     SessionParams=get_s_params_session(Session),
%     FileInstr=get_fileinstr_session(Session), % How to fetch files when finished.
%     case inviso:start_session() of
% 	{ok,{SessionID,Result}} ->           % Session started.
% 	    {ok,SessionID,Result,FileInstr};
% 	{error,Reason} ->                    % Some general failure.
% 	    {error,Reason}
%     end;
% h_run_suite_start_session(CtrlNode,Session) ->
%     Nodes=get_nodes_session(Session),        % Get all desired Erlang nodes.
%     SessionParams=get_s_params_session(Session),
%     FileInstr=get_fileinstr_session(Session), % How to fetch files when finished.
%     case node() of
% 	CtrlNode ->                          % We are at the control node, no rpc.
% 	    case inviso:start_session(Nodes) of
% 		{ok,{SessionID,Result}} ->   % Session started at control component.
% 		    {ok,SessionID,Result,FileInstr};
% 		{error,Reason} ->            % Some general failure.
% 		    {error,Reason}
% 	    end;
% 	_ ->                                 % We are at other node than control component.
% 	    case rpc:call(CtrlNode,inviso,start_session,[Nodes]) of
% 		{ok,{SessionID,Result}} ->   % Session started at control component.
% 		    {ok,SessionID,Result};
% 		{error,Reason} ->
% 		    {error,Reason};
% 		{badrpc,Reason} ->
% 		    {error,{badrpc,{CtrlNode,Reason}}}
% 	    end
%     end.
%% ------------------------------------------------------------------------------

%% ------------------------------------------------------------------------------
%% Help functions for handle_info.
%% ------------------------------------------------------------------------------

%% Help function which traverses the NodeData structure and tries to connect/start
%% runtime components on all nodes where we not already have one.
%% NodeData contains the current status (according to internal status notation).
%% In that we can find the nodes that are not running.
%% Returns {ok,NewNodeData} or 'false' if no changes have been made. Can also
%% return {error,Reason} if serious problem was encountered.
%% If a "serious" problem occurred we may very well be in an undefined state.
%% Meaning we have added nodes we don't want and are unable to get rid of.
timeout_check_new_nodes(CtrlNode,NodeData,RTtag,Dbg) ->
    timeout_check_new_nodes_2(CtrlNode,ld_nodedata_mk_list(NodeData),RTtag,Dbg,[],false).
 
timeout_check_new_nodes_2(CtrlNode,[X={Node,Status,SID}|Rest],RTtag,Dbg,Accum,Flag) ->
    case ld_nodedata_is_running(Status) of
	false ->                             % Then try to start it!
	    case timeout_check_new_nodes_3(CtrlNode,Node,RTtag,Dbg) of
		{ok,NewStatus} ->            % Managed to start it this time.
		    timeout_check_new_nodes_2(CtrlNode,Rest,RTtag,Dbg,
					      [{Node,NewStatus,SID}|Accum],true);
		false ->                     % No still no success.
		    timeout_check_new_nodes_2(CtrlNode,Rest,RTtag,Dbg,
					      [{Node,Status,SID}|Accum],Flag);
		{error,Reason} ->            % Serious problem, cancel the operation.
		    {error,Reason}
	    end;
	true ->                              % Running! No need to start it then.
	    timeout_check_new_nodes_2(CtrlNode,Rest,RTtag,Dbg,[X|Accum],Flag) % Do nothing.
    end;
timeout_check_new_nodes_2(_,[],_RTtag,_Dbg,Accum,true) -> % There has been some change, report it.
    {ok,ld_nodedata_mk_struct(lists:reverse(Accum))};
timeout_check_new_nodes_2(_,[],_RTtag,_Dbg,_Accum,false) -> % No changes, drop Accum.
    false.

timeout_check_new_nodes_3(void,_Node,RTtag,Dbg) -> % The non-distributed case.
    case inviso:add_node(trace_c_tag(RTtag)) of
	{ok,{{tag,Tag},_State}} when Tag=/=RTtag -> % We don't want this one.
	    case inviso:stop_node() of
		ok ->
		    inviso_tool_lib:debug(stop_wrong_tag,Dbg,[local_runtime,ok]),
		    false;
		{error,Reason} ->            % Strange could not stop the node!
		    inviso_tool_lib:debug(stop_wrong_tag,Dbg,[local_runtime,{error,Reason}]),
		    {error,{stop_node,local_runtime}}
	    end;
	{ok,already_added} ->                % We expect a connect event then.
	    false;                           % Wait for it instead.
	{ok,Result} ->
	    {ok,ld_nodedata_translate_to_status(Result)};
	{error,Reason} ->
	    inviso_tool_lib:debug(add_node,Dbg,[local_runtime,{error,Reason}]),
	    false
    end;
timeout_check_new_nodes_3(CtrlNode,Node,RTtag,Dbg) ->
    case inviso_tool_lib:inviso_cmd(CtrlNode,add_nodes,[[Node],trace_c_tag(RTtag)]) of
	{ok,[{Node,{ok,already_added}}]} ->  % Strange, must have added it self.
	    false;                           % Wait for the event message instead.
	{ok,[{Node,{error,_Reason}}]} ->     % Can still not start this runtime comp.
	    false;
	{ok,[{Node,{ok,{{tag,Tag},_State}}}]} when Tag=/=RTtag -> % Don't want this one!
	    case inviso_tool_lib:inviso_cmd(CtrlNode,stop_nodes,[[Node]]) of
		{ok,_} ->                    % Managed to stop it.
		    inviso_tool_lib:debug(stop_wrong_tag,Dbg,[[Node],ok]),
		    false;                   % Indicate it as no started node!
		{error,Reason} ->            % This is really difficult.
		    inviso_tool_lib:debug(stop_wrong_tag,Dbg,[[Node],{error,Reason}]),
		    {error,{stop_nodes,Reason}}
	    end;
	{ok,[{Node,Result}]} ->              % We managed to get control over it!
	    {ok,ld_nodedata_translate_to_status(Result)};
	{error,Reason} ->                    % Strange, some kind of general failure.
	    inviso_tool_lib:debug(add_nodes,Dbg,[[Node],{error,Reason}]),
	    {error,{add_nodes,Reason}}
    end.
%% ------------------------------------------------------------------------------


%% Help function which changes the internal status for a node in the nodes
%% list found the loop data.
%% Returns {ok,NewNodeData} or 'false' if no changes were made to NodeData.
trace_event_disconnect_node(Node,NodeData) ->
    case ld_nodedata_get_status(Node,NodeData) of
	{ok,_Status} ->                      % It is one of our nodes!
	    {ok,ld_nodedata_update_status(Node,ld_nodedata_disconnected_status(),NodeData)};
	false ->                             % No it is not our node.
	    false
    end.
%% ------------------------------------------------------------------------------

%% Help function which takes a nodedata structure of nodes from the Loop data
%% structure and marks a node as running. Note that we will only do that
%% for a node which is part of our set of nodes.
%% Returns 'false' or {ok,NewNodeData}.
trace_event_connect_node(Node,NodeData) ->
    case ld_nodedata_get_status(Node,NodeData) of
	{ok,_Status} ->                      % It is one of our nodes!
	    {ok,ld_nodedata_update_status(Node,ld_nodedata_running_status(),NodeData)};
	false ->                             % No it is not our node.
	    false
    end.
%% ------------------------------------------------------------------------------

%% Help function which compares two tags to find out if it is one "we" have
%% generated.
%% Returns 'true' or 'false'.
trace_event_is_my_tag(MyTag,MyTag) ->
    true;
trace_event_is_my_tag(_,_MyTag) ->
    false.
%% ------------------------------------------------------------------------------


%% Help function which sends an event to our subscriber. Note that this function
%% must handle the case when there is no subscriber registered.
%% Returns nothing significant.
handle_send_event(Pid,What) when pid(Pid) ->
    Pid ! {inviso_tool_event,self(),now(),What};
handle_send_event(_,_) ->                    % The case when no subscriber.
    true.
%% ------------------------------------------------------------------------------



%% ==============================================================================
%% Various help functions.
%% ==============================================================================


%% Help function which returns the tag with which we cn identify if a runtime
%% component was previously controlled by "us" or not.
trace_c_tag(RTtag) ->
    RTtag.
%% -----------------------------------------------------------------------------

%% Help function which takes a return value from add_nodes and stops all
%% runtimes that were adopted. I.e they were already running and had another
%% reference tag than our own. If the keep connecting mechanism is activated
%% the stopped nodes may be connected later.
%% Returns a new Result list with the stopped nodes removed.
stop_adopted_nodes(NodeName,RTtag,Result,Dbg) ->
    Nodes=lists:foldl(fun({N,{ok,{adopted,_S,_Status,Tag}}},Acc) when Tag=/=RTtag -> [N|Acc];
			 (_,Acc) -> Acc
		      end,
		      [],
		      Result),               % Get all "problem" nodes.
    case inviso_tool_lib:inviso_cmd(NodeName,stop_nodes,[Nodes]) of
	{ok,StopResult} ->
	    inviso_tool_lib:debug(stop_wrong_tags,Dbg,[Result,StopResult]),
	    Result2=lists:filter(fun({N,{ok,_}}) ->
					 case lists:member(N,Nodes) of
					     true ->  % Remove this node.
						 false;
					     false -> % Keep in result list.
						 true
					 end;
				    (_) ->   % Keep it in all other cases.
					 true
				 end,
				 Result),    % A Result without the "wrong" nodes.
	    {ok,Result2};
	{error,Reason} ->
	    inviso_tool_lib:debug(stop_wrong_tags,Dbg,[Result,{error,Reason}]),
	    {error,Reason}
    end.
%% -----------------------------------------------------------------------------

%% Help function which of the nodes in NodeParams finds out which nodes are not
%% already active in a session.
%% Returns {NodeInASession,NodesNotInASession}.
%% Note that this function can only be used for the distributed situation.
%% NodeParams=[{Node,TracerData},...]
not_in_session(NodeDataList,NodeParams) ->
    not_in_session_2(NodeDataList,NodeParams,[],[]).
	    
not_in_session_2(NodeDataList,[NodeParam|Rest],AccIn,AccNotIn)
  when tuple(NodeParam),size(NodeParam)>1 ->
    Node=element(1,NodeParam),
    case lists:keysearch(Node,1,NodeDataList) of
	{value,{_,_,SID}} when pid(SID) ->   % This one is in a session already.
	    not_in_session_2(NodeDataList,Rest,[Node|AccIn],AccNotIn);
	_ ->                                 % Not in a session, add to accum.
	    not_in_session_2(NodeDataList,Rest,AccIn,[Node|AccNotIn])
    end;
not_in_session_2(NodeDataList,[_|Rest],AccIn,AccNotIn) -> % Don't understand.
    not_in_session_2(NodeDataList,Rest,AccIn,AccNotIn); % Skip it!
not_in_session_2(_,[],AccIn,AccNotIn) ->
    {lists:reverse(AccIn),lists:reverse(AccNotIn)}.
%% -----------------------------------------------------------------------------


%% -----------------------------------------------------------------------------
%% Functions handling start options.
%% -----------------------------------------------------------------------------

-define(SAFETY_CATCHES,safety_catches).      % Own defined safety catches.
-define(KEEP_CONNECTING,keep_connecting).    % Periodically try to connect.
-define(SUBSCRIBE_EVENTS,subscribe_events).  % Have the tool inform someone about events.
-define(DEBUG,debug).                        % Make the tool print own debug info.
-define(RTTAG,rtref).                        % The reference used for our nodes.

-define(DBG_OFF,off).                        % No internal debug indicator.

%% This function extracts options given to the start-functions.
%% Returns {ok,SafetyCatches,TimeOutMillisec,EventPid,Dbg} or {error,Reason}.
get_and_check_init_options(Opts) when list(Opts) ->
    case (catch get_and_check_init_options_2(Opts)) of
	{'EXIT',Reason} ->
	    exit(Reason);
	Result ->
	    Result
    end;
get_and_check_init_options(_) ->
    {error,not_a_list}.

get_and_check_init_options_2(Opts) ->
    SafetyCatches=
	case lists:keysearch(?SAFETY_CATCHES,1,Opts) of
	    {value,{_,SCs}} when list(SCs) ->
		case check_safetycatches(SCs) of
		    ok ->
			SCs;
		    {error,Reason} ->        % Then we will abort everything.
			throw({error,{bad_safetycatch,Reason}})
		end;
	    {value,{_,SC={_,_}}} ->          % Lets tolerate a single tuple too.
		SC;                          % We already made the check, return it!
	    _ ->                             % No safety!
		[]
	end,
    TimeOut=                                 % Shall we contineously connect to nodes?
	case lists:member(?KEEP_CONNECTING,Opts) of
	    true ->                          % Default connect interval.
		?DEFAULT_TIMEOUT;
	    false ->
		case lists:keysearch(?KEEP_CONNECTING,1,Opts) of
		    {value,{_,T}} when integer(T) ->
			T;
		    _ ->                     % Do not use serverside timeouts!
			infinity
		end
	end,
    EventPid=                                % Process subscribing to events from tool.
	case lists:keysearch(?SUBSCRIBE_EVENTS,1,Opts) of
	    {value,{_,Pid}} when pid(Pid) ->
		Pid;
	    _ ->
		void
	end,
    Dbg=
	case lists:keysearch(?DEBUG,1,Opts) of
	    {value,{_,Level}} ->
		{level,Level};
	    _ ->
		?DBG_OFF                     % Indicates that internal debug is off!
	end,
    RTtag=
	case lists:keysearch(?RTTAG,1,Opts) of
	    {value,{_,Tag}} ->               % The reference used to indicate that
		Tag;                         % it is our runtime component.
	    false ->
		inviso_standard_ref
	end,
    {ok,SafetyCatches,TimeOut,EventPid,Dbg,RTtag}.
%% -----------------------------------------------------------------------------

%% Help function which just checks that it is a list of two-tuples with atoms.
check_safetycatches([{M,F}|Rest]) when atom(M),atom(F) ->
    check_safetycatches(Rest);
check_safetycatches([Item|_]) ->
    {error,Item};
check_safetycatches([]) ->
    ok.
%% -----------------------------------------------------------------------------


%% -----------------------------------------------------------------------------
%% Functions handling loop-data
%% -----------------------------------------------------------------------------

%% Loopdata is a record:
%% #ld{
%%     ctrlnode     : The Erlang node where the control component runs, or 'void'
%%                    {node,NodeName} or {{M,F,Args},NodeName}.
%%     ctrlpid      : The pid of the control component.
%%     safetycatches: A list of {Mod,Func} of all safety catch functions.
%%     nodedata     : A datastructure decribing all our runtime components and
%%                    their current status. See below separate primitives.
%%     timeout      : gen_server server side timeout for the inviso tool.
%%     eventpid     : Subscriber to events generated by the tool.
%%     dbg          : inviso tool internal debug flag.
%%     rttag        : The tag indicating our trace run-time component processes.
%%    }

%% Macro constants used to highlight that we are dependant of the return
%% values from trace control component APIs.
-record(ld,{ctrlnode,ctrlpid,safetycatches,nodedata,
	    timeout,eventpid,dbg,rttag,sessions}).

mk_ld(CtrlNode,Pid,Result,SafetyCatches,TimeOut,EventPid,Dbg,RTtag) ->
    NodeData=ld_nodedata_mk_nodedata(Result),
    #ld{
       ctrlnode=CtrlNode,
       ctrlpid=Pid,
       safetycatches=SafetyCatches,
       nodedata=NodeData,
       timeout=TimeOut,
       eventpid=EventPid,
       dbg=Dbg,
       rttag=RTtag,
       sessions=[]
       }.
%% -----------------------------------------------------------------------------

%% Primitive fetching the nodename component of the ctrlnode field.
get_ctrlnode_node_ld(#ld{ctrlnode={_,NodeName}}) -> NodeName;
get_ctrlnode_node_ld(#ld{ctrlnode=void}) -> void.
%% -----------------------------------------------------------------------------

get_ctrlpid_ld(#ld{ctrlpid=CtrlPid}) -> CtrlPid.
%% -----------------------------------------------------------------------------

get_catches_ld(#ld{safetycatches=SC}) -> SC.
%% -----------------------------------------------------------------------------

%% Primitive fetching the NodeData structure.
get_nodedata_ld(#ld{nodedata=NodeData}) -> NodeData.
put_nodedata_ld(NodeData,LD) -> LD#ld{nodedata=NodeData}.
%% -----------------------------------------------------------------------------

get_timeout_ld(#ld{timeout=TimeOut}) -> TimeOut.
%% -----------------------------------------------------------------------------

get_eventpid_ld(#ld{eventpid=EventPid}) -> EventPid.
%% -----------------------------------------------------------------------------

get_dbg_ld(#ld{dbg=DBG}) -> DBG.
%% -----------------------------------------------------------------------------

get_rttag_ld(#ld{rttag=RTtag}) -> RTtag.
%% -----------------------------------------------------------------------------

%% Function adding a new session to the loop data structure.
%% Note that adding a session involves putting it in the sessions list but
%% also updating all nodedata.
%% NodeData is a list of all nodes which possibly can be part of the session.
%% That means that the SID may be set for a runtime component node that is not
%% yet connected!
add_session_ld(Nodes,SID,LD) ->
    NewNodeData=add_session_ld_2(Nodes,SID,ld_nodedata_mk_list(LD#ld.nodedata),[]),
    LD#ld{nodedata=ld_nodedata_mk_struct(NewNodeData),
	  sessions=[SID|LD#ld.sessions]}.

add_session_ld_2(Nodes,SID,[X={Node,Status,_SID}|Rest],Accum) ->
    case lists:member(Node,Nodes) of
	true ->                              % Set the SID for this node!
	    add_session_ld_2(Nodes,SID,Rest,[{Node,Status,SID}|Accum]);
	false ->
	    add_session_ld_2(Nodes,SID,Rest,[X|Accum])
    end;
add_session_ld_2(_,_,[],Accum) ->
    lists:reverse(Accum).
%% -----------------------------------------------------------------------------

%% Function which removes a session from the loopdata structure.
%% It both removes it from the nodedata structure and from the sessions list.
%% Returns {NewLD,Nodes} where Nodes is a list of all nodes changed in the
%% nodedata structure.
delete_session_ld(SID,LD) ->
    {NewNodeDataList,Nodes}=
	delete_session_ld_2(SID,ld_nodedata_mk_list(get_nodedata_ld(LD)),[],[]),
    NewSessions=lists:delete(SID,get_sessions_ld(LD)),
    {LD#ld{nodedata=ld_nodedata_mk_struct(NewNodeDataList),sessions=NewSessions},
     Nodes}.

delete_session_ld_2(SID,[{Node,Status,SID}|Rest],AccNDL,AccNodes) ->
    delete_session_ld_2(SID,Rest,[{Node,Status,void}|AccNDL],[Node|AccNodes]);
delete_session_ld_2(SID,[NodeDataTuple|Rest],AccNDL,AccNodes) ->
    delete_session_ld_2(SID,Rest,[NodeDataTuple|AccNDL],AccNodes);
delete_session_ld_2(_,[],AccNDL,AccNodes) ->
    {lists:reverse(AccNDL),AccNodes}.
%% -----------------------------------------------------------------------------

%% Returns a list of all existing sessions and their SIDs. A list of
%% [SID,...].
get_sessions_ld(#ld{sessions=Sessions}) -> Sessions.
%% -----------------------------------------------------------------------------


%% -----------------------------------------------------------------------------
%% Functions working on our internal representation of runtime nodes, node_data.
%% -----------------------------------------------------------------------------
%% The nodedata structure is currently a list of {Node,Status,SID} or {Status,SID}
%% in the non-distributed case. where Status describes the current condition of
%% the node. SID is a pid if the node currently belongs to a session 
%% (then handled by that session handler pid).
%% All nodes which have been specified to be owned by our trace control component
%% will be part of the nodedata structure.
%%
%% A node can be in one of these states:
%% (1) running       : Normal, "we" started a fresh runtime here.
%% (2) rejected      : Runtime already controlled by someone else.
%% (3) {error,Reason}: Unavailable due to some reason.
%%
%% It is not the work of the inviso tool master process to know in what state a runtime
%% component is. That is the work of a session handler process!


%% Help function which converts a list returned from the trace control components
%% start function. Returns a nodedata data structure which is supposed to be
%% stored in the loopdata field nodedata.
%% Note that this function my only be used on a return value comming from a call
%% where we do not accept runtime components started with another tag than our own.
%% This since there is no mechanism which converts such returned statuses to an
%% {error,Reason} instead.
ld_nodedata_mk_nodedata(Result) ->
    lists:map(fun({N,Status}) ->
		      {N,ld_nodedata_translate_to_status(Status),void}
	      end,
	      Result).

%% Help function which converts the status value returned from the trace control
%% component when starting a runtime component to our internal status values.
ld_nodedata_translate_to_status({error,refused}) -> % The rt is owned by other.
    rejected;
ld_nodedata_translate_to_status({error,Reason}) -> {error,Reason};
ld_nodedata_translate_to_status({ok,new}) -> % Never a problem when freshly started.
    ld_nodedata_running_status();
ld_nodedata_translate_to_status({ok,{adopted,_State,_Status,_Tag}}) ->
    ld_nodedata_running_status();
ld_nodedata_translate_to_status({ok,already_added}) -> % We will get a connect event.
    {error,already_added}.                   % Therefore set it in error for now.
%% -----------------------------------------------------------------------------

%% Function which returns the current status for a certain Node.
%% Returns {ok,Status} or 'false' if Node does not exist in the NodeData structure.
ld_nodedata_get_status(Node,NodeData) ->
    case lists:keysearch(Node,1,NodeData) of
	{value,{_,Status,_SID}} ->
	    {ok,Status};
	false ->
	    false
    end.
%% -----------------------------------------------------------------------------

%% Function changing the status of and existing node.
%% Returns a new nodedata structure.
ld_nodedata_update_status(NodeName,Status,NodeData) ->
    case lists:keysearch(NodeName,1,NodeData) of
	{value,{_,_,SID}} ->
	    lists:keyreplace(NodeName,1,NodeData,{NodeName,Status,SID});
	false ->                             % Then no use changing it.
	    NodeData
    end.
%% -----------------------------------------------------------------------------

%% Function which removes a session identifier pid from the nodedata structure.
%% This is necessary when a session has been ended and the SID no longer exists.
%% Returns {NewNodeData,Nodes} where Nodes is a list of all nodes that were
%% updated.
ld_nodedata_remove_sid(SID,NodeData) ->
    {NewNDList,Nodes}=ld_nodedata_remove_sid_2(SID,ld_nodedata_mk_list(NodeData),[],[]),
    {ld_nodedata_mk_struct(NewNDList),Nodes}.

ld_nodedata_remove_sid_2(SID,[{Node,Status,SID}|Rest],AccND,AccNodes) ->
    ld_nodedata_remove_sid_2(SID,Rest,[{Node,Status,void}|AccND],[Node|AccNodes]);
ld_nodedata_remove_sid_2(SID,[NodeDataTuple|Rest],AccND,AccNodes) ->
    ld_nodedata_remove_sid_2(SID,Rest,[NodeDataTuple|AccND],AccNodes);
ld_nodedata_remove_sid_2(_,[],AccND,AccNodes) ->
    {lists:reverse(AccND),AccNodes}.
%% -----------------------------------------------------------------------------

%% Function which takes a nodedata structure and returns a list of
%% {Node,Status,SID}.
ld_nodedata_mk_list(NodeData) -> NodeData.
%% -----------------------------------------------------------------------------

%% Function which takes a nodedata list and returns a nodedata structure
%% as it is supposed to be stored in the loopdata structure.
ld_nodedata_mk_struct(NodeList) -> NodeList.
%% -----------------------------------------------------------------------------

%% This function translates the internal status indication to the one used
%% in the nodes/0 api.
ld_nodedata_translate_to_ui_indication(running) -> running;
ld_nodedata_translate_to_ui_indication(_) -> unavailable.
%% -----------------------------------------------------------------------------

%% Help function which returns the internal status indication for a running
%% runtime component.
ld_nodedata_running_status() -> running.
%% -----------------------------------------------------------------------------

%% Help function which returns the internal status indication for a disconnected
%% runtime component.
ld_nodedata_disconnected_status() -> {error,disconnected}.
%% -----------------------------------------------------------------------------

%% Help function which returns 'true' or 'false' depending on whether the
%% argument indicates a node to be running or not.
ld_nodedata_is_running(running) -> true;
ld_nodedata_is_running(_) -> false.
%% -----------------------------------------------------------------------------
