1 'use strict';
  2 
  3 // ================================================================
  4 //  Code related to monitoring the fad system
  5 // ================================================================
  6 
  7 var incomplete = 0;
  8 
  9 sub_incomplete.onchange = function(evt)
 10 {
 11     if (!evt.data)
 12         return;
 13 
 14     var inc = evt.obj['incomplete'];
 15     if (!inc || inc>0xffffffffff)
 16         return;
 17 
 18     if (incomplete>0)
 19         return;
 20 
 21     if (dim.state("MCP").name!="TakingData")
 22         return;
 23 
 24     console.out("");
 25     dim.log("Incomplete event ["+inc+","+incomplete+"] detected, sending MCP/STOP");
 26 
 27     incomplete = inc;
 28     dim.send("MCP/STOP");
 29 }
 30 
 31 // ================================================================
 32 //  Code related to taking data
 33 // ================================================================
 34 
 35 /**
 36  * reconnect to problematic FADs
 37  *
 38  * Dis- and Reconnects to FADs, found to be problematic by call-back function
 39  * onchange() to have a different CONNECTION value than 66 or 67. 
 40  * 
 41  * @returns
 42  *      a boolean is returned. 
 43  *      reconnect returns true if:
 44  *          * nothing needed to be reset --> no problems found by onchange()
 45  *          * the reconnection went fine.
 46  *      
 47  *      reconnect *never returns false* so far.
 48  *
 49  * @example
 50  *      if (!sub_connections.reconnect())
 51  *          exit();
 52  */
 53 function reconnect(list, txt)
 54 { /*
 55     var reset = [ ];
 56 
 57     for (var i=0; i<list.length; i++)
 58         {
 59             console.out("  FAD %2d".$(list[i])+" lost during "+txt);
 60             reset.push(parseInt(list[i]/10));
 61         }
 62 
 63     reset = reset.filter(function(elem,pos){return reset.indexOf(elem)==pos;});
 64 
 65     console.out("");
 66     console.out("  FADs belong to crate(s): "+reset);
 67     console.out("");
 68 */
 69     console.out("");
 70     dim.log("Trying automatic reconnect ["+txt+",n="+list.length+"]...");
 71 
 72     if (list.length>3)
 73         throw new Error("Too many boards to be reconnected. Please check what happened.");
 74 
 75     for (var i=0; i<list.length; i++)
 76     {
 77         console.out("   ...disconnect "+list[i]);
 78         dim.send("FAD_CONTROL/DISCONNECT", list[i]);
 79     }
 80 
 81     console.out("   ...waiting for 3s");
 82     v8.sleep(3000);
 83 
 84     for (var i=0; i<list.length; i++)
 85     {
 86         console.out("   ...reconnect "+list[i]);
 87         dim.send("FAD_CONTROL/CONNECT", list[i]);
 88     }
 89 
 90     console.out("   ...waiting for 1s");
 91 
 92     // Wait for one second to bridge possible pending connects
 93     v8.sleep(1000);
 94 
 95     console.out("   ...checking connection");
 96 
 97     // Wait for FAD_CONTROL to realize that all boards are connected
 98     // FIXME: Wait for '40' boards being connected instead
 99     try
100     {
101         dim.wait("FAD_CONTROL", "Connected", 3000);
102     }
103     catch (e)
104     {
105         if (dim.state("FAD_CONTROL").name!="Connecting")
106         {
107             console.out("");
108             console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
109             console.out("");
110             throw e;
111         }
112 
113         var crates = [];
114         for (var i=0; i<list.length; i++)
115             crates[list[i]/10] = true;
116 
117         include('scripts/crateReset.js');
118         crateReset(crates);
119     }
120 
121     // Wait also for MCP to have all boards connected again
122     dim.wait("MCP", "Idle", 3000);
123 
124     dim.log("Automatic reconnect successfull.");
125     console.out("");
126 }
127 
128 function takeRun(type, count, time)
129 {
130     if (!count)
131         count = -1;
132     if (!time)
133         time = -1;
134 
135     var custom = typeof(type)=="function";
136 
137     var nextrun = sub_startrun.get().obj['next'];
138     dim.log("Take run %3d".$(nextrun)+": N="+count+" T="+time+"s ["+(custom?"custom":type)+"]");
139 
140     // FIXME: Replace by callback?
141     //
142     // DN: I believe instead of waiting for 'TakingData' one could split this
143     // up into two checks with an extra condition:
144     //  if type == 'data':
145     //      wait until ThresholdCalibration starts:
146     //          --> this time should be pretty identical for each run
147     //      if this takes longer than say 3s:
148     //          there might be a problem with one/more FADs
149     //    
150     //      wait until "TakingData":
151     //          --> this seems to take even some minutes sometimes... 
152     //              (might be optimized rather soon, but still in the moment...)
153     //      if this takes way too long: 
154     //          there might be something broken, 
155     //          so maybe a very high time limit is ok here.
156     //          I think there is not much that can go wrong, 
157     //          when the Thr-Calib has already started. Still it might be nice 
158     //          If in the future RateControl is written so to find out that 
159     //          in case the threshold finding algorithm does 
160     //          *not converge as usual*
161     //          it can complain, and in this way give a hint, that the weather
162     //          might be a little bit too bad.
163     //  else:
164     //      wait until "TakingData":
165     //          --> in a non-data run this time should be pretty short again
166     //      if this takes longer than say 3s:
167     //          there might be a problem with one/more FADs
168     //  
169 
170     // Use this if you use the rate control to calibrate by rates
171     //if (!dim.wait("MCP", "TakingData", -300000) )
172     //{
173     //    throw new Error("MCP took longer than 5 minutes to start TakingData"+
174     //                    "maybe this idicates a problem with one of the FADs?");
175     //}
176 
177     // ================================================================
178     //  Function for Critical voltage
179     // ================================================================
180 
181     // INSTALL a watchdog... send FAD_CONTROL/CLOSE_OPEN_FILES
182     // could send MCP/RESET as well but would result in a timeout
183     var callback = dim.onchange['FEEDBACK'];
184     dim.onchange['FEEDBACK'] = function(state)
185     {
186         if (callback)
187             callback.call(this, state);
188 
189         if ((state.name=="Critical" || state.name=="OnStandby") &&
190             (this.last!="Critical"  && this.last!="OnStandby"))
191         {
192             console.out("Feedback state changed from "+this.last+" to "+state.name+" [takeRun.js]");
193 
194             // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
195             dim.send("MCP/STOP");
196         }
197 
198         this.last=state.name;
199     }
200 
201     // Here we could check and handle fad losses
202 
203     incomplete = 0;
204 
205     var start = true;
206 
207     for (var n=0; n<3; n++)
208     {
209         if (start)
210         {
211             dim.send("MCP/START", time, count, custom?"custom":type);
212             if (custom)
213                 type();
214         }
215 
216         try
217         {
218             dim.wait("MCP", "TakingData", 15000);
219             break;
220         }
221         catch (e)
222         {
223             if (dim.state("MCP").name=="TriggerOn" &&
224                 dim.state("FAD_CONTROL").name=="Connected" &&
225                 dim.state("FTM_CONTROL").name=="TriggerOn")
226             {
227                 console.out("");
228                 console.out("Waiting for TakingData timed out. Everything looks ok, but file not yet open... waiting once more.");
229                 start = false;
230                 continue;
231             }
232 
233             start = true;
234 
235             console.out("");
236             console.out(" + MCP:         "+dim.state("MCP").name);
237             console.out(" + FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
238             console.out(" + FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
239             console.out("");
240 
241             if (dim.state("MCP").name!="Configuring3" ||
242                 (dim.state("FAD_CONTROL").name!="Configuring1" &&
243                  dim.state("FAD_CONTROL").name!="Configuring2"))
244                 throw e;
245 
246             console.out("");
247             console.out("Waiting for fadctrl to get configured timed out... checking for in-run FAD loss.");
248 
249             var con  = sub_connections.get();
250             var stat = con.obj['status'];
251 
252             console.out("Sending MCP/RESET");
253             dim.send("MCP/RESET");
254 
255             dim.wait("FTM_CONTROL", "Valid",     3000);
256             dim.wait("FAD_CONTROL", "Connected", 3000);
257             dim.wait("MCP",         "Idle",      3000);
258 
259             var list = [];
260             for (var i=0; i<40; i++)
261                 if (stat[i]!=0x43)
262                     list.push(i);
263 
264             reconnect(list, "configuration");
265 
266             if (n==2)
267                 throw e;
268 
269             //dim.wait("MCP", "Idle", 3000);
270         }
271     }
272 
273     // This is to check if we have missed the event. This can happen as
274     // a race condition when the MCP/STOP is sent by the event handler
275     // but the run was not yet fully configured.
276     var statefb = dim.state("FEEDBACK").name;
277     if (statefb=="Critical" || statefb=="OnStandby")
278     {
279         console.out("Run started by FEEDBACK in state "+statefb);
280         dim.send("MCP/STOP"); // Includes FAD_CONTROL/CLOSE_ALL_OPEN_FILES
281 
282         dim.onchange['FEEDBACK'] = callback;
283 
284         return true;
285     }
286 
287     dim.wait("MCP", "Idle", time>0 ? time*1250 : undefined); // run time plus 25%
288 
289     // REMOVE watchdog
290     dim.onchange['FEEDBACK'] = callback;
291 
292     if (incomplete)
293     {
294         console.out("");
295         console.out(" - MCP:         "+dim.state("MCP").name);
296         console.out(" - FAD_CONTROL: "+dim.state("FAD_CONTROL").name);
297         console.out(" - FTM_CONTROL: "+dim.state("FTM_CONTROL").name);
298 
299         dim.wait("FTM_CONTROL", "Valid",     3000);
300         dim.wait("FAD_CONTROL", "Connected", 3000);
301         dim.wait("MCP",         "Idle",      3000);
302 
303         var str = incomplete.toString(2);
304         var len = str.length;
305 
306         var list = [];
307         for (var i=0; i<str.length; i++)
308             if (str[str.length-i-1]=='1')
309                 list.push(i);
310 
311         reconnect(list, "data taking");
312 
313         return false;
314     }
315 
316     // FIXME: What if the ext1 is not enabled in the configuration?
317     if (type=="data")
318     {
319         var dim_trg = new Subscription("FAD_CONTROL/TRIGGER_COUNTER");
320         var counter = dim_trg.get(3000);
321 
322         // The check on physics and pedestal triggers is to ensure that
323         // there was at least a chance to receive any event (e.g. in case
324         // of an interrupt this might not be the case)
325         if (counter.qos!=111 &&
326             (counter.data['N_trg']>1000 || counter.data['N_ped']>5) &&
327             counter.data['N_ext1']==0) // 'o' for open
328             throw new Error("No ext1 triggers received during data taking... please check the reason and report in the logbook.");
329         dim_trg.close();
330     }
331 
332     return true;
333 }
334