dsCreateBatch

CREATEBATCH - create and submit jobs to run sets of simulations or analyses.

 Usage:
   studyinfo=dsCreateBatch(model,varargin)

 Inputs:
   - DynaSim model (see dsGenerateModel)
   - modifications_set (as returned by dsVary2Modifications())
     - options:
       'compile_flag'  : whether to compile simulation using coder instead of
                         interpreting Matlab {0 or 1} (default: 0)
       'verbose_flag'  : whether to display informative messages/logs (default: 0)
       'overwrite_flag': whether to overwrite existing data files {0 or 1} (default: 0)
     - options for cluster computing:
       'sims_per_job'  : number of simulations to run per cluster job (default: 1)
       'memory_limit'  : memory to allocate per cluster job (default: '8G')
       'batch_dir'     : where to save job scripts
       'qsub_mode'     : whether to use SGE -t array for 1 qsub, mode: 'array'; or
                         qsub in csh for loop, mode: 'loop'. (default: 'loop').
     - options for parallel computing: (requires Parallel Computing Toolbox)
       - Note: parallel computing has been DISABLED for debugging...
       'parallel_flag' : whether to use parfor to run simulations {0 or 1} (default: 0)
       'num_cores'     : number of cores to specify in the parallel pool

 Outputs:
   - DynaSim studyinfo (see dsCheckStudyinfo for schema info)

 Dependencies: dsSetupStudy, dsUpdateStudy

 See also: dsGenerateModel, dsSimulate, dsCheckStudyinfo, dsVary2Modifications

 Author: Jason Sherfey, PhD <jssherfey@gmail.com>
 Copyright (C) 2016 Jason Sherfey, Boston University, USA

0001 function [studyinfo, cmd] = dsCreateBatch(base_model,modifications_set,varargin)
0002 %CREATEBATCH - create and submit jobs to run sets of simulations or analyses.
0003 %
0004 % Usage:
0005 %   studyinfo=dsCreateBatch(model,varargin)
0006 %
0007 % Inputs:
0008 %   - DynaSim model (see dsGenerateModel)
0009 %   - modifications_set (as returned by dsVary2Modifications())
0010 %     - options:
0011 %       'compile_flag'  : whether to compile simulation using coder instead of
0012 %                         interpreting Matlab {0 or 1} (default: 0)
0013 %       'verbose_flag'  : whether to display informative messages/logs (default: 0)
0014 %       'overwrite_flag': whether to overwrite existing data files {0 or 1} (default: 0)
0015 %     - options for cluster computing:
0016 %       'sims_per_job'  : number of simulations to run per cluster job (default: 1)
0017 %       'memory_limit'  : memory to allocate per cluster job (default: '8G')
0018 %       'batch_dir'     : where to save job scripts
0019 %       'qsub_mode'     : whether to use SGE -t array for 1 qsub, mode: 'array'; or
0020 %                         qsub in csh for loop, mode: 'loop'. (default: 'loop').
0021 %     - options for parallel computing: (requires Parallel Computing Toolbox)
0022 %       - Note: parallel computing has been DISABLED for debugging...
0023 %       'parallel_flag' : whether to use parfor to run simulations {0 or 1} (default: 0)
0024 %       'num_cores'     : number of cores to specify in the parallel pool
0025 %
0026 % Outputs:
0027 %   - DynaSim studyinfo (see dsCheckStudyinfo for schema info)
0028 %
0029 % Dependencies: dsSetupStudy, dsUpdateStudy
0030 %
0031 % See also: dsGenerateModel, dsSimulate, dsCheckStudyinfo, dsVary2Modifications
0032 %
0033 % Author: Jason Sherfey, PhD <jssherfey@gmail.com>
0034 % Copyright (C) 2016 Jason Sherfey, Boston University, USA
0035 
0036 %% check inputs
0037 options=dsCheckOptions(varargin,{...
0038   'compile_flag',0,{0,1},...
0039   'parallel_flag',0,{0,1},...
0040   'num_cores',4,[],... % # cores for parallel processing (SCC supports 1-12)
0041   'sims_per_job',1,[],... % how many sims to run per cluster job
0042   'memory_limit','8G',[],... % how much memory to allocate per cluster job
0043   'batch_dir',[],[],...
0044   'simulator_options',[],[],...
0045   'verbose_flag',0,{0,1},...
0046   'overwrite_flag',0,{0,1},...
0047   'process_id',[],[],... % process identifier for loading studyinfo if necessary
0048   'qsub_mode','loop',{'loop','array'},... % whether to submit jobs as an array using qsub -t or in a for loop
0049   'one_solve_file_flag',0,{0,1},... % use only 1 solve file of each type, but can't vary mechs yet
0050   'solver','rk4',{'euler','rk1','rk2','rk4','modified_euler','rungekutta','rk','ode23','ode45',...
0051     'ode1','ode2','ode3','ode4','ode5','ode8','ode113','ode15s','ode23s','ode23t','ode23tb'},... % DynaSim and built-in Matlab solvers
0052   'auto_gen_test_data_flag',0,{0,1},...
0053   'unit_test_flag',0,{0,1},...
0054   },false);
0055 
0056 %% auto_gen_test_data_flag argin
0057 if options.auto_gen_test_data_flag
0058   varargs = varargin;
0059   varargs{find(strcmp(varargs, 'auto_gen_test_data_flag'))+1} = 0;
0060   varargs(end+1:end+2) = {'unit_test_flag',1};
0061   argin = [{base_model},{modifications_set}, varargs]; % specific to this function
0062 end
0063 
0064 %% Main function.
0065 %% Set up studyinfo structure, study directory and output file names
0066 [studyinfo,options.simulator_options]=dsSetupStudy(base_model,'modifications_set',modifications_set,'simulator_options',options.simulator_options,'process_id',options.process_id);
0067 study_file=fullfile(studyinfo.study_dir,'studyinfo.mat');
0068 num_simulations=length(modifications_set);
0069 
0070 %% check whether study has already completed
0071 % if options.overwrite_flag==0
0072   [~,s]=dsMonitorStudy(studyinfo.study_dir,'verbose_flag',0,'process_id',options.process_id);
0073   if s==1 % study finished
0074     if options.verbose_flag
0075       fprintf('Study already finished. Not creating new batch.\n');
0076     end
0077 
0078     studyinfo=dsCheckStudyinfo(studyinfo.study_dir,'process_id',options.process_id, varargin{:});
0079 
0080     return;
0081   end
0082 % end
0083 
0084 %% Check if attempting to compile Experiment
0085 if isa(options.simulator_options.experiment,'function_handle') && options.compile_flag
0086   options.compile_flag=0;
0087   options.simulator_options.compile_flag=0;
0088   wprintf('solver compilation is not available for experiments run on the cluster.');
0089 end
0090 
0091 %% create base solve_file (m- or MEX-file)
0092 solve_file = dsGetSolveFile(base_model,studyinfo,options.simulator_options);
0093 
0094 %% add appropriate MEX extension if compiled
0095 %   NOTE: the extension depends on whether a 32-bit or 64-bit machine is used
0096 if options.compile_flag
0097   % get directory where solve_file is located and the file name
0098   [fpath,fname]=fileparts2(solve_file);
0099 
0100   % get list of files in solve directory
0101   D=dir(fpath);
0102 
0103   % get extension of files with matching names
0104   tmp=regexp({D.name},[fname '\.(\w+)$'],'tokens','once');
0105   tmp=unique([tmp{:}]);
0106 
0107   % append extension to solve_file if regular mex (not non supported matlab solver mex)
0108   if ~any(strcmp(options.solver, {'ode113','ode15s','ode23s','ode23t','ode23tb'})) % not mex supported)
0109     full_solve_file=[solve_file '.' tmp{1}];
0110 
0111     % remove '_mex' suffix from solve_file for compatibility with dsGetSolveFile()
0112     solve_file=regexp(solve_file,'(.+)_mex$','tokens','once');
0113     solve_file=solve_file{1};
0114   else
0115     full_solve_file=solve_file;
0116   end
0117 else
0118   full_solve_file=solve_file;
0119 end
0120 studyinfo.base_solve_file=solve_file;
0121 
0122 %% set name of batch_dir for this study
0123 % get study-specific timestamp
0124 % timestamp = datestr(studyinfo.study_id,'yyyymmddHHMMSS');
0125 
0126 % get home directory of the current user
0127 [~,home]=system('echo $HOME');
0128 
0129 %% create batch directory
0130 study_dir_full_path = fileparts2(studyinfo.study_dir); % convert to path with closing slash
0131 [~, study_dir_name, study_dir_suffix] = fileparts(study_dir_full_path); % get final directory
0132 study_dir_name=[study_dir_name study_dir_suffix];
0133 
0134 if ~options.auto_gen_test_data_flag && ~options.unit_test_flag
0135   main_batch_dir = fullfile(strtrim(home),'batchdirs');
0136 
0137   % create main batch_dir
0138   if ~exist(main_batch_dir,'dir')
0139     mkdir(main_batch_dir);
0140   end
0141 
0142   batch_dir = fullfile(main_batch_dir,study_dir_name);
0143   %batch_dir = fullfile(strtrim(home),'batchdirs',['Batch' timestamp]);
0144 else
0145   study_dir = fileparts2(studyinfo.study_dir);
0146   [~, rel_study_dir] = fileparts(study_dir);
0147   batch_dir = fullfile(rel_study_dir,'batchdirs',study_dir_name);
0148 end
0149 
0150 if options.verbose_flag
0151   fprintf('\nPREPARING BATCH:\n');
0152 end
0153 
0154 %% create batch_dir to hold m-file jobs and more for this study
0155 if ~exist(batch_dir,'dir')
0156   if options.verbose_flag
0157     fprintf('Creating batch jobs directory: %s\n',batch_dir);
0158   end
0159   mkdir(batch_dir);
0160 end
0161 
0162 %% copy study_file to batchdir
0163 [success,msg]=copyfile(study_file,batch_dir);
0164 if ~success
0165   error(msg)
0166 end
0167 
0168 %% locate DynaSim toolbox
0169 dynasim_path = dsGetRootPath(); % root is one level up from directory containing this function
0170 dynasim_functions=fullfile(dynasim_path,'functions');
0171 
0172 %% locate mechanism files
0173 [mech_paths,mech_files]=dsLocateModelFiles(base_model);
0174 
0175 %% add paths to studyinfo structure
0176 studyinfo.paths.dynasim_functions = dynasim_functions;
0177 studyinfo.paths.mechanisms = mech_paths;
0178 studyinfo.paths.batch_dir = batch_dir;
0179 
0180 %% edit simulator_options so that subsequent single simulations do not create further batches
0181 options.simulator_options.parallel_flag = 0;
0182 options.simulator_options.cluster_flag = 0;
0183 options.simulator_options.verbose_flag = 1; % turn on verbose for cluster logs (stdout)
0184 
0185 % collect paths to add to all jobs
0186 % reason: adding paths to jobs supports m-files stored/associated with mechanism files
0187 % add dynasim root path (where functions are located)
0188 addpaths=cat(2,dynasim_path,regexp(genpath(dynasim_functions),':','split'));
0189 %addpaths={dynasim_path,dynasim_functions};
0190 % add the toolbox models directory and all subdirectories
0191   % note: users can store their models as subdirectories of dynasim/models
0192   % and incorporate them in their models without worrying about paths.
0193 addpaths=cat(2,addpaths,fullfile(dynasim_path,'models'));
0194 %addpaths=regexp(genpath(dynasim_path),':','split');
0195 if ~isempty(mech_paths)
0196   addpaths=cat(2,addpaths,mech_paths);
0197   addpaths=unique(addpaths);
0198 end
0199 
0200 %% create jobs (k)
0201 jobs={};
0202 skip_sims=[];
0203 sim_cnt=0;
0204 num_jobs=ceil(num_simulations/options.sims_per_job);
0205 
0206 if options.verbose_flag && ~options.one_solve_file_flag
0207   fprintf('Creating %g cluster jobs in batch directory for %g simulations...\n',num_jobs,num_simulations);
0208 end
0209 
0210 if ~options.one_solve_file_flag
0211   for k = 1:num_jobs
0212     job_file=fullfile(batch_dir,sprintf('sim_job%g.m',k));
0213     sim_ids=sim_cnt+(1:options.sims_per_job);
0214     sim_ids=sim_ids(sim_ids<=num_simulations);
0215     sim_cnt=sim_cnt+options.sims_per_job;
0216 
0217     % only run simulations if data_file does not exist (unless overwrite_flag=1)
0218     proc_sims=[]; % simulations to run in this job
0219     for j=1:length(sim_ids)
0220       if (options.simulator_options.save_data_flag && ~exist(studyinfo.simulations(sim_ids(j)).data_file,'file')) || ...
0221           (options.simulator_options.save_results_flag && ~exist(studyinfo.simulations(sim_ids(j)).result_files{1},'file')) || ...
0222           options.overwrite_flag
0223         proc_sims=[proc_sims sim_ids(j)];
0224       end
0225     end
0226 
0227     if isempty(proc_sims)
0228       skip_sims=[skip_sims sim_ids];
0229       continue; % skip this job
0230     else
0231       skip_sims=[skip_sims setdiff(sim_ids,proc_sims)];
0232     end
0233 
0234     WriteSimJob(proc_sims,job_file); % create job script
0235 
0236     jobs{end+1}=job_file;
0237 
0238     % add job_file to studyinfo
0239     for j=1:length(proc_sims)
0240       studyinfo.simulations(proc_sims(j)).job_file=job_file;
0241     end
0242 
0243   end %for
0244 else %one_solve_file_flag
0245   job_file=fullfile(batch_dir, 'sim_job.m'); %always use this file
0246 
0247   WriteSimJob([], job_file); % create job script
0248 
0249   % TODO: compile job_file
0250 %   if options.compile_flag
0251 %     job_file = dsPrepareMEX(job_file, options); %compile the job file
0252 %   end
0253 
0254   % add job_file to studyinfo
0255   [studyinfo.simulations.job_file] = deal(job_file); %same job file for all sims
0256 end %if
0257 
0258 % TODO: have job scripts create lock for each sim; then, check for lock
0259 %   file in this function and skip simulations if they're already running
0260 %   (similar to skipping if data_file already exists).
0261 
0262 %% exit if there are no simulations to run
0263 if numel(skip_sims)==num_simulations
0264   if options.verbose_flag
0265     fprintf('Data exists for all simulations in study. nothing to do.\n');
0266   end
0267 
0268   return;
0269 end
0270 
0271 %% create script_filename (list of vars or jobs)
0272 if strcmp(options.qsub_mode, 'loop')
0273   script_filename = fullfile(batch_dir,'scriptlist.txt');
0274   if options.verbose_flag
0275     fprintf('Creating file listing jobs in batch directory: %s\n',script_filename);
0276   end
0277 
0278   % write to file
0279   fScript=fopen(script_filename,'wt');
0280 
0281   for j=1:length(jobs)
0282     [~,thisFilename]=fileparts2(jobs{j});
0283     fprintf(fScript,'%s\n',thisFilename);
0284   end
0285 
0286   fclose(fScript);
0287 end
0288 
0289 if ~options.one_solve_file_flag
0290   % copy solve_file to each sim-specific solve sub-directory: <study_dir>/solve/sim<k>/<solve_file>
0291   %   and set sim-specific studyinfo
0292   if options.verbose_flag
0293     fprintf('Creating distinct solver sub-directories for %g simulations...\n',num_simulations);
0294   end
0295   %[solve_path,solve_name,solve_ext]=fileparts2(solve_file);
0296   [solve_path,solve_name,solve_ext]=fileparts2(full_solve_file);
0297 
0298   % prepare studyinfo with simulation-specific metadata
0299   for sim=1:num_simulations
0300     if ismember(sim,skip_sims)
0301       continue; % do nothing with this simulation
0302     end
0303 
0304     this_solve_path=fullfile(solve_path,['sim' num2str(sim)]);
0305 
0306     if ~exist(this_solve_path,'dir')
0307       %if options.verbose_flag
0308       %  fprintf('creating solver sub-directory for simulation #%g: %s\n',sim,this_solve_path);
0309       %end
0310       mkdir(this_solve_path);
0311     end
0312 
0313     [success,msg]=copyfile(full_solve_file,this_solve_path); % creates solve sub-directory and copies the base solve file to it
0314 
0315     if ~success, error(msg); end
0316 
0317     % set studyinfo solve_file to use for this simulation
0318     this_solve_file=fullfile(this_solve_path,[solve_name solve_ext]);
0319     studyinfo.simulations(sim).solve_file=this_solve_file;
0320     studyinfo.simulations(sim).batch_dir=batch_dir;
0321     studyinfo.simulations(sim).simulator_options=options.simulator_options;
0322 
0323     % set solve_file for this simulation (will be passed to dsSimulate as an option)
0324     if isa(options.simulator_options.experiment,'function_handle')
0325       studyinfo.simulations(sim).simulator_options.solve_file=[];
0326     else
0327       studyinfo.simulations(sim).simulator_options.solve_file=this_solve_file;
0328     end
0329 
0330     % set vary to [] to avoid each sim expanding to a new set
0331     studyinfo.simulations(sim).simulator_options.vary=[];
0332     studyinfo.simulations(sim).simulator_options.sim_id=studyinfo.simulations(sim).sim_id;
0333     studyinfo.simulations(sim).error_log='';
0334 
0335   end %sim
0336 
0337   % copy studyinfo file to batch_dir for each simulation
0338   for sim=1:num_simulations
0339     this_study_file=fullfile(batch_dir,sprintf('studyinfo_%g.mat',sim));
0340 
0341     if sim==1
0342       try
0343         save(this_study_file,'studyinfo','-v7');
0344         if ~strcmp(reportUI,'matlab')
0345           [wrn_msg,wrn_id] = lastwarn;
0346           if strcmp(wrn_msg,'save: wrong type argument ''function handle''')
0347             error('save: wrong type argument ''function handle''');
0348           end
0349         end
0350       catch
0351         fprintf('Data is not ''-v7'' compatible. Saving in hdf5 format.\n')
0352         save(this_study_file,'studyinfo','-hdf5');
0353       end
0354       first_study_file=this_study_file;
0355     else
0356       % use copyfile() after saving first b/c >10x faster than save()
0357       [success,msg]=copyfile(first_study_file,this_study_file);
0358 
0359       if ~success, error(msg); end
0360     end
0361   end %sim
0362 
0363 else %one_solve_file_flag
0364   % set studyinfo solve_file to use for this simulation
0365   [studyinfo.simulations.solve_file] = deal(full_solve_file);
0366   [studyinfo.simulations.batch_dir] = deal(batch_dir);
0367   [studyinfo.simulations.simulator_options] = deal(options.simulator_options);
0368 
0369   % set vary to [] to avoid each sim expanding to a new set
0370   for iSim = 1:num_simulations
0371     studyinfo.simulations(iSim).simulator_options.vary = [];
0372     studyinfo.simulations(iSim).simulator_options.sim_id = studyinfo.simulations(iSim).sim_id;
0373   end
0374   [studyinfo.simulations.error_log] = deal('');
0375 
0376   % copy studyinfo file to batch_dir since more information now
0377   batch_study_file = fullfile(batch_dir,'studyinfo.mat');
0378   try
0379     save(batch_study_file,'studyinfo','-v7');
0380     if ~strcmp(reportUI,'matlab')
0381       [wrn_msg,wrn_id] = lastwarn;
0382       if strcmp(wrn_msg,'save: wrong type argument ''function handle''')
0383         error('save: wrong type argument ''function handle''');
0384       end
0385     end
0386   catch
0387     fprintf('Data is not ''-v7'' compatible. Saving in hdf5 format.\n')
0388     save(batch_study_file,'studyinfo','-hdf5');
0389   end
0390 end
0391 
0392 %% update studyinfo on disk
0393 dsStudyinfoIO(studyinfo,study_file,options.simulator_options.sim_id,options.verbose_flag);
0394 
0395 % TODO: remove old lock files ..
0396 
0397 %% check for qsub on system
0398 [status,result]=system('which qsub');
0399 
0400 if options.auto_gen_test_data_flag || options.unit_test_flag
0401   status = 0;
0402   result = 1;
0403 end
0404 
0405 if isempty(result)
0406   [~,host] = system('hostname');
0407   fprintf('qsub not found on host (%s).\n',strtrim(host));
0408   fprintf('Jobs NOT submitted to cluster queue.\n');
0409 else % on cluster with qsub
0410   % submit jobs (e.g., fScripjobs_memlimit batch_dir 32G)
0411   % check status of study
0412   [~,s]=dsMonitorStudy(studyinfo.study_dir,'verbose_flag',0);
0413 
0414   if s~=1 % study not finished
0415     % submit jobs using shell script
0416     if options.auto_gen_test_data_flag || options.unit_test_flag
0417       batch_dir = rel_study_dir;
0418     end
0419 
0420     [batch_dir_path,batch_dir_name,batch_suffix]=fileparts(batch_dir);
0421     batch_dir_name=[batch_dir_name batch_suffix];
0422 
0423     if ~options.auto_gen_test_data_flag && ~options.unit_test_flag
0424       dsFnPath = fileparts(mfilename('fullpath'));
0425     else
0426       dsFnPath = 'dsFnPath';
0427     end
0428 
0429     if options.parallel_flag
0430       warning('jobs in the cluster use a single thread')
0431     end
0432 
0433     if strcmp(options.qsub_mode, 'array') && ~options.one_solve_file_flag
0434       % TODO: remove old error and output files; put e and o in their own dirs
0435       cmd = sprintf('echo ''%s/qsub_jobs_array %s sim_job'' | qsub -V -hard -l ''h_vmem=%s'' -wd %s -N %s_sim_job -t 1-%i',...
0436         dsFnPath, batch_dir, options.memory_limit, batch_dir, batch_dir_name, num_jobs);
0437     elseif strcmp(options.qsub_mode, 'array') && options.one_solve_file_flag
0438       [~, job_filename] = fileparts2(job_file); %remove path and extension
0439       cmd = sprintf('echo ''%s/qsub_jobs_array_one_file %s %s'' | qsub -V -hard -l ''h_vmem=%s'' -wd %s -N %s_sim_job -t 1-%i:%i',...
0440         dsFnPath, batch_dir, job_filename, options.memory_limit, batch_dir, batch_dir_name, num_simulations, options.sims_per_job);
0441       % NOTE: using num_simulations, not num_jobs, since the job_file will
0442       %   determine it's own sims to run
0443     elseif strcmp(options.qsub_mode, 'loop')
0444       if strcmp(reportUI,'matlab')
0445         ui_command = '"matlab -nodisplay -singleCompThread -r"';
0446         l_directives = ['-l mem_total=',options.memory_limit];
0447       else
0448         ui_command = '"octave-cli --eval"';
0449         l_directives = ['-l centos7=TRUE -l mem_total=',options.memory_limit];
0450       end
0451       cmd = sprintf('%s/qsub_jobs_loop %s ''%s'' ''%s''',dsFnPath,batch_dir_name,ui_command,l_directives);
0452     end
0453 
0454     % add shell script to linux path if not already there
0455     setenv('PATH', [getenv('PATH') ':' dynasim_functions ':' fullfile(dynasim_functions, 'internal')]);
0456 
0457     if options.verbose_flag
0458       fprintf('Submitting cluster jobs with shell command: "%s"\n',cmd);
0459     end
0460 
0461     if ~options.auto_gen_test_data_flag && ~options.unit_test_flag
0462       [status,result]=system(cmd);
0463     end
0464 
0465     if status > 0
0466       if options.verbose_flag
0467         fprintf('Submit command failed: "%s"\n',cmd);
0468         disp(result);
0469       end
0470       return;
0471     else
0472       if options.verbose_flag
0473         fprintf('Submit command status: \n');
0474         disp(result);
0475       end
0476     end
0477 
0478     if options.verbose_flag
0479       %fprintf('%g jobs successfully submitted!\ntip: use dsMonitorStudy(''%s'') or dsMonitorStudy(studyinfo) to track status of cluster jobs.\n',num_jobs,studyinfo.study_dir);
0480       fprintf('%g jobs successfully submitted!\n',num_jobs);
0481     end
0482   elseif s==1 % study is finished
0483     if options.verbose_flag
0484       fprintf('Study already finished. not submitting jobs.\n');
0485     end
0486   end
0487 end
0488 
0489 %% auto_gen_test_data_flag argout
0490 if options.auto_gen_test_data_flag
0491   dirIn = studyinfo.study_dir;
0492   removeStudyinfo()
0493 
0494   if ~isempty(studyinfo)
0495     studyinfo = [];
0496   end
0497   argout = {studyinfo, cmd}; % specific to this function
0498 
0499   dsUnitSaveAutoGenTestDir(argin, argout, [], dirIn);
0500 end
0501 
0502 %% unit test
0503 if options.unit_test_flag
0504   % remove fields that cause issues in unit testing
0505   removeStudyinfo()
0506   if ~isempty(studyinfo)
0507     studyinfo = [];
0508   end
0509 end
0510 
0511 
0512 
0513 %% NESTED FUNCTIONS
0514   function WriteSimJob(sim_ids,job_file)
0515     % purpose: write m-file to job_file to run simulations of sim_ids
0516 
0517     % create job file
0518     fjob=fopen(job_file,'wt');
0519 
0520     % load studyinfo using helper function to avoid busy file errors
0521     %fprintf(fjob,'studyinfo=dsCheckStudyinfo(''%s'',''process_id'',%g);\n',study_file,sim_ids(1), varargin{:});
0522     %fprintf(fjob,'load(''%s'',''studyinfo'');\n',study_file);
0523 
0524     [~, job_filename] = fileparts(job_file); %remove path and extension
0525 
0526     if ~options.one_solve_file_flag
0527       % function declaration
0528       fprintf(fjob, 'function %s\n\n', job_filename);
0529 
0530       % set IDs of simulations to run
0531       fprintf(fjob,'SimIDs=[%s]\n',num2str(sim_ids));
0532     else %only 1 file
0533       % function declaration
0534       fprintf(fjob, 'function %s(simIDstart, simIDstep, simIDlast)\n\n', job_filename);
0535 
0536       if options.compile_flag
0537         fprintf(fjob, 'assert(isa(simIDstart, ''double''));\n');
0538         fprintf(fjob, 'assert(isa(simIDstep, ''double''));\n');
0539         fprintf(fjob, 'assert(isa(simIDlast, ''double''));\n');
0540       end
0541 
0542       % set IDs of simulations to run
0543       fprintf(fjob,'SimIDs = simIDstart:min(simIDlast,simIDstart+simIDstep-1);\n');
0544     end
0545 
0546     % add paths
0547     for p=1:length(addpaths)
0548       if ~isempty(addpaths{p})
0549         fprintf(fjob,'addpath %s\n',addpaths{p});
0550       end
0551     end
0552 
0553     % loop over and run simulations in this job
0554     if options.parallel_flag
0555       % set parallel computing options
0556       %fprintf(fjob,'started=0;\npool=gcp(''nocreate'');\n');
0557       %fprintf(fjob,'if isempty(pool), parpool(%g); started=1; end\n',options.num_cores);
0558       fprintf(fjob,'c=parcluster;\nsaveAsProfile(c,''local_%g'');\nparpool(c,%g);\n',k,options.num_cores);
0559 
0560       % use parfor loop
0561       fprintf(fjob,'parfor s=1:length(SimIDs)\n');
0562     else
0563       % use for loop
0564       fprintf(fjob,'for s=1:length(SimIDs)\n');
0565     end
0566 
0567     fprintf(fjob,'\tSimID=SimIDs(s);\n');
0568 
0569     % each job should have try-catch to capture studyinfo.simulations(k).error_log
0570     fprintf(fjob,'\ttry\n');
0571 
0572     % load studyinfo for this simulation
0573     if ~options.one_solve_file_flag
0574       fprintf(fjob,'\t\tload(fullfile(''%s'',sprintf(''studyinfo_%%g.mat'',SimID)),''studyinfo'');\n',batch_dir);
0575     else
0576       fprintf(fjob,'\t\tload(fullfile(''%s'',''studyinfo.mat''),''studyinfo'');\n',batch_dir);
0577     end
0578     % compare paths between compute machine and studyinfo startup
0579     fprintf(fjob,'\t\t[valid,message]=dsCheckHostPaths(studyinfo);\n');
0580 
0581     if ~options.one_solve_file_flag
0582       fprintf(fjob,'\t\tif ~valid\n\t\t  lasterr(message);\n\t\t  for s=1:length(SimIDs), dsUpdateStudy(studyinfo.study_dir,''sim_id'',SimIDs(s),''status'',''failed''); end\n\t\t  continue;\n\t\tend\n');
0583     end
0584 
0585     % simulate model with proper modifications and options
0586     fprintf(fjob,'\t\tsiminfo=studyinfo.simulations(SimID);\n');
0587     fprintf(fjob,'\t\toptions=rmfield(siminfo.simulator_options,{''modifications'',''studyinfo'',''analysis_functions'',''plot_functions'',''sim_id''});\n');
0588     fprintf(fjob,'\t\tkeyvals=dsOptions2Keyval(options);\n');
0589     fprintf(fjob,'\t\tfprintf(''-----------------------------------------------------\\n'');\n');
0590     fprintf(fjob,'\t\tfprintf(''Processing simulation %%g (%%g of %%g in this job)...\\n'',SimID,s,length(SimIDs));\n');
0591     fprintf(fjob,'\t\tfprintf(''-----------------------------------------------------\\n'');\n');
0592     fprintf(fjob,'\t\tdata=dsSimulate(studyinfo.base_model,''modifications'',siminfo.modifications,''studyinfo'',studyinfo,''sim_id'',SimID,keyvals{:});\n');
0593     fprintf(fjob,'\t\tfor i=1:length(siminfo.result_functions)\n');
0594     fprintf(fjob,'\t\t\tdsAnalyze(data,siminfo.result_functions{i},''result_file'',siminfo.result_files{i},''save_data_flag'',1,siminfo.result_options{i}{:});\n');
0595     fprintf(fjob,'\t\tend\n');
0596 
0597     % add error handling
0598     fprintf(fjob,'\tcatch err\n');
0599 
0600     %fprintf(fjob,'\t\ttry delete(fullfile(studyinfo.study_dir,[''.locked_'' num2str(SimID)])); end\n');
0601     fprintf(fjob,'\t\tdisplayError(err);\n');
0602     fprintf(fjob,'\tend\n');
0603 
0604     % end loop over simulations in this job
0605     fprintf(fjob,'end\n');
0606     if options.parallel_flag
0607       %fprintf(fjob,'if started, delete(gcp); end\n');
0608       fprintf(fjob,'delete(gcp)\n');
0609     end
0610 
0611     % exit script
0612     [status,result]=system('which qsub');
0613     if ~isempty(result) % on cluster
0614       fprintf(fjob,'exit\n');
0615     end
0616 
0617     % close job file
0618     fclose(fjob);
0619   end % WriteSimJob
0620 
0621   function removeStudyinfo()
0622     % Problem: studyinfo files have many timestamps and absolute paths
0623     % Solution: remove studyinfo files
0624 
0625     studyDirFiles = rls(studyinfo.study_dir);
0626     studyinfoFiles = studyDirFiles(~cellfun(@isempty, strfind(studyDirFiles, 'studyinfo')));
0627     for k = 1:length(studyinfoFiles)
0628       thisFile = studyinfoFiles{k};
0629       delete(thisFile)
0630     end
0631   end
0632 
0633 end % main fn
0634 
0635 % -------------------
0636 % in dsCreateBatch():
0637 % -------------------
0638 % create solve_file
0639 % for each sim k
0640 %   copy to this_solve_file = /solve/sim<k>/solve_file
0641 %   set studyinfo.simulations(k).solve_file=this_solve_file
0642 % ...
0643 % in sim job.m:
0644 % dsSimulate(model{k},'solve_file',solve_file{k},...)
0645 % where solve_file{k}=studyinfo.simulations(k).solve_file