dsMonitorStudy

MONITORSTUDY - display information on study progress.

 Usage:
   [studyinfo,status]=dsMonitorStudy(studyinfo,key/value options)

 Inputs:
   - studyinfo: DynaSim studyinfo structure, study directory, or studyinfo MAT filename
   - options: TODO

 Outputs:
   - studyinfo: DynaSim studyinfo structure
   - status: numeric code
       0 (study in progress)
       1 (study finished)
       2 (error in study)
       -1 (function failed)

 See also: dsSimulate, dsCreateBatch, dsCheckStudyinfo

 Author: Jason Sherfey, PhD <jssherfey@gmail.com>
 Copyright (C) 2016 Jason Sherfey, Boston University, USA

0001 function [studyinfo,study_status] = dsMonitorStudy(studyinfo,varargin)
0002 %MONITORSTUDY - display information on study progress.
0003 %
0004 % Usage:
0005 %   [studyinfo,status]=dsMonitorStudy(studyinfo,key/value options)
0006 %
0007 % Inputs:
0008 %   - studyinfo: DynaSim studyinfo structure, study directory, or studyinfo MAT filename
0009 %   - options: TODO
0010 %
0011 % Outputs:
0012 %   - studyinfo: DynaSim studyinfo structure
0013 %   - status: numeric code
0014 %       0 (study in progress)
0015 %       1 (study finished)
0016 %       2 (error in study)
0017 %       -1 (function failed)
0018 %
0019 % See also: dsSimulate, dsCreateBatch, dsCheckStudyinfo
0020 %
0021 % Author: Jason Sherfey, PhD <jssherfey@gmail.com>
0022 % Copyright (C) 2016 Jason Sherfey, Boston University, USA
0023 
0024 % Check inputs
0025 options=dsCheckOptions(varargin,{...
0026   'verbose_flag',1,{0,1},...
0027   'process_id',[],[],... % process identifier for loading studyinfo if necessary
0028   },false);
0029 if isstruct(studyinfo) && isfield(studyinfo,'study_dir')
0030   % retrieve most up-to-date studyinfo structure from studyinfo.mat file
0031   studyinfo=dsCheckStudyinfo(studyinfo.study_dir,'process_id',options.process_id, varargin{:});
0032 else
0033   % process the provided studyinfo structure
0034   studyinfo=dsCheckStudyinfo(studyinfo,'process_id',options.process_id, varargin{:});
0035 end
0036 
0037 % Check status of study
0038 if all(strcmp('finished',{studyinfo.simulations.status}))
0039   study_status=1; % study finished
0040 elseif any(~cellfun(@isempty,{studyinfo.simulations.error_log}))
0041   study_status=-1; % errors in study
0042 else
0043   study_status=0; % study in progress
0044 end
0045 
0046 if options.verbose_flag==0
0047   return;
0048 end
0049 
0050 fprintf('-------------------------------------------------------------\n');
0051 %% 1.0 Processing statistics by host (e.g., compute times)
0052 % get host for each simulation
0053 running=find(~arrayfun(@(x)isempty(x.machine_info),studyinfo.simulations));
0054 if any(running)
0055   hosts=arrayfun(@(x)x.machine_info.host_name,studyinfo.simulations(running),'uni',0);
0056   % make list of unique hosts
0057   uniq_hosts=unique(hosts);
0058   num_hosts=length(uniq_hosts);
0059   % collect info for each host
0060   num_simulations=zeros(1,num_hosts);
0061   num_finished=zeros(1,num_hosts);
0062   mean_duration=zeros(1,num_hosts);
0063   num_running=zeros(1,num_hosts);
0064   num_failed=zeros(1,num_hosts);
0065   num_cores=zeros(1,num_hosts);
0066   for i=1:num_hosts
0067     % get list of simulations on this host
0068     these_sims=running(strcmp(uniq_hosts{i},hosts));
0069     % store the number of simulations processed on this host
0070     num_simulations(i)=length(these_sims);
0071     % get list of simulations that are running on this host
0072     started=strcmp('started',{studyinfo.simulations(these_sims).status});
0073     num_running(i)=length(find(started));
0074     % get list of simulations that have finished
0075     finished=strcmp('finished',{studyinfo.simulations(these_sims).status});
0076     num_finished(i)=length(find(finished));
0077     % get mean duration of simulations that have finished
0078     if num_finished(i)>0
0079       mean_duration(i)=mean([studyinfo.simulations(these_sims(finished)).duration]);
0080     else
0081       mean_duration(i)=nan;
0082     end
0083     % get list of simulations that have failed
0084     failed=strcmp('failed',{studyinfo.simulations(these_sims).status});
0085     num_failed(i)=length(find(failed));
0086     % tech info
0087     try
0088       num_cores(i)=studyinfo.simulations(these_sims(1)).machine_info.num_cores;
0089     catch
0090       num_cores(i)=nan;
0091     end
0092     %total_memory=studyinfo.simulations(these_sims(1)).machine_info.total_memory; % string
0093   end
0094   % sort hosts by mean_duration
0095   [~,I]=sort(mean_duration,2,'descend');
0096   % display info for each host
0097   fprintf('Processing statistics (hosts sorted by mean compute time T):\n');
0098   for i=1:num_hosts
0099     index=I(i);
0100     fprintf('  @%s (%g cores)\n',uniq_hosts{index},num_cores(index));%,num_finished(index),num_simulations(index),mean_duration(index),num_failed(index),num_running(index));
0101     fprintf('    %g of %g sims finished (T: %gsec); %g failed; %g running.\n',num_finished(index),num_simulations(index),mean_duration(index),num_failed(index),num_running(index));
0102   end
0103 end
0104 
0105 %% 2.0 Errors
0106 % get list of errors over all simulations
0107 errors={studyinfo.simulations.error_log};
0108 % collapse into list of unique errors
0109 uniq_errors=unique(errors(cellfun(@ischar,errors)));
0110 % number of unique errors
0111 num_uniq_errors=length(uniq_errors);
0112 % indices to simulations with errors
0113 error_inds=find(~cellfun(@isempty,errors)); 
0114 if options.verbose_flag
0115   % Display errors for each simulation
0116   if any(error_inds) % some sims had errors
0117     fprintf('Errors:\n');
0118     for i=1:length(error_inds)
0119       siminfo=studyinfo.simulations(error_inds(i));
0120       if strcmp(siminfo.status,'finished')
0121         fprintf('  Simulation %g (error corrected, now %s):\n',siminfo.sim_id,siminfo.status);
0122       elseif ~strcmp(siminfo.status,'failed')
0123         fprintf('  Simulation %g (now re-%s):\n',siminfo.sim_id,siminfo.status);
0124       else
0125         fprintf('  Simulation %g (%s):\n',siminfo.sim_id,siminfo.status);
0126       end
0127       try fprintf('    Host name: %s\n',siminfo.machine_info.host_name); end
0128       fprintf('    Start time: %s\n',siminfo.start_time);
0129       fprintf('    Error log: %s\n',siminfo.error_log);
0130     end
0131   end
0132 else
0133   % Display each unique error message only once
0134   if any(error_inds) % some sims had errors
0135     fprintf('Unique Errors:\n');
0136     % display each unique error message once and list the simulation IDs with
0137     % matching errors
0138     for i=1:num_uniq_errors
0139       % do nothing if there is no error message
0140       if isempty(uniq_errors{i})
0141         continue;
0142       end
0143       % get list of simulations with this error
0144       matches=strcmp(uniq_errors{i},errors);
0145       sim_ids=[studyinfo.simulations(matches).sim_id];
0146       fprintf('  Simulation(s) %s:\n',num2str(sim_ids));
0147       fprintf('    Error log: %s\n',uniq_errors{i});
0148     end
0149   end
0150 end
0151 
0152 %% 3.0 Paths
0153 fprintf('Paths:\n');
0154 fprintf('  Study directory: %s\n',studyinfo.study_dir);
0155 if ~isempty(studyinfo.paths)
0156   if isfield(studyinfo.paths,'mechanisms') && iscell(studyinfo.paths.mechanisms)
0157     if length(studyinfo.paths.mechanisms)==1 % print path on one line
0158       fprintf('  Model files:     %s\n',studyinfo.paths.mechanisms{1});
0159     else % print each mech path on a separate line
0160       fprintf('  Model files:\n');
0161       for i=1:length(studyinfo.paths.mechanisms)
0162         fprintf('    %s\n',studyinfo.paths.mechanisms{i});
0163       end
0164     end
0165   end
0166   if isfield(studyinfo.paths,'dynasim_functions')
0167     fprintf('  DynaSim functions: %s\n',studyinfo.paths.dynasim_functions);
0168   end
0169   if isfield(studyinfo.paths,'batch_dir')
0170     fprintf('  Batch directory: %s\n',studyinfo.paths.batch_dir);
0171   end
0172 end
0173 
0174 %% 4.0 Status summary
0175 % get status for each simulation
0176 status={studyinfo.simulations.status};
0177 % make list of status types
0178 uniq_status=unique(status);
0179 % display counts for each status type
0180 fprintf('Simulation status summary:\n');
0181 for i=1:length(uniq_status)
0182   count=length(find(strcmp(uniq_status{i},status)));
0183   fprintf('  %g %s\n',count,uniq_status{i});
0184 end
0185 
0186 %% notify about special cases
0187 if all(strcmp('finished',status))
0188   fprintf('**** ALL SIMULATIONS HAVE FINISHED ****\n');
0189 end
0190 if all(strcmp('initialized',status))
0191   fprintf('**** NO SIMULATIONS HAVE STARTED ****\n');
0192 end
0193 if all(strcmp('started',status))
0194   fprintf('**** ALL SIMULATIONS ARE RUNNING ****\n');
0195 end
0196 if all(strcmp('failed',status))
0197   fprintf('**** ALL SIMULATIONS FAILED ****\n');
0198 end
0199 
0200 fprintf('-------------------------------------------------------------\n');