ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/UserCode/benhoob/cmsnotes/StopSearch/systematics.tex
(Generate patch)

Comparing UserCode/benhoob/cmsnotes/StopSearch/systematics.tex (file contents):
Revision 1.15 by claudioc, Thu Oct 11 07:33:42 2012 UTC vs.
Revision 1.18 by claudioc, Fri Oct 12 04:55:23 2012 UTC

# Line 145 | Line 145 | comes from data/MC statistics.  This
145   result directly in a 3\% uncertainty on the dilepton BG, which is by far
146   the most important one.
147  
148 + \subsection{Uncertainty from MC statistics}
149 + This affects mostly the \ttll\ background estimate, which is taken
150 + from
151 + Monte Carlo with appropriate correction factors.  This uncertainty
152 + is negligible in the low \met\ signal regions, and grows to about
153 + 15\% in SRG.
154  
149 \subsection{Uncertainty on the \ttll\ Acceptance}
155  
156 < [CLAUDIO: WE NEED TO DISCUSS THIS A LITTLE MORE -- THEN I CAN PUT THE
152 < WORDS IN]
156 > \subsection{Uncertainty on the \ttll\ Acceptance}
157  
158   The \ttbar\ background prediction is obtained from MC, with corrections
159   derived from control samples in data. The uncertainty associated with
# Line 189 | Line 193 | The variations considered are
193    This effect was studied earlier using 7~TeV samples and found to be negligible.
194    \end{itemize}
195  
196 <
197 < \begin{table}[!h]
198 < \begin{center}
199 < {\footnotesize
200 < \begin{tabular}{l||c||c|c|c|c|c|c|c}
201 < \hline
202 < Sample              & Powheg & Madgraph & Mass Up & Mass Down & Scale Up & Scale Down &
203 < Match Up & Match Down \\
204 < \hline
205 < \hline
206 < SRA      & $579 \pm 10$ & $569 \pm 16$ & $591 \pm 18$ & $610 \pm 22$ & $651 \pm 22$ & $537 \pm 16$ & $578 \pm 18$ & $570 \pm 17$  \\
207 < \hline
208 < SRB      & $328 \pm 7$ & $307 \pm 11$ & $329 \pm 13$ & $348 \pm 15$ & $344 \pm 15$ & $287 \pm 10$ & $313 \pm 13$ & $307 \pm 12$  \\
209 < \hline
210 < SRC      & $111 \pm 4$ & $99 \pm 5$ & $107 \pm 7$ & $113 \pm 8$ & $124 \pm 8$ & $95 \pm 6$ & $93 \pm 6$ & $106 \pm 6$  \\
211 < \hline
212 < SRD      & $39 \pm 2$ & $35 \pm 3$ & $41 \pm 4$ & $41 \pm 5$ & $47 \pm 5$ & $33 \pm 3$ & $31 \pm 3$ & $39 \pm 4$  \\
213 < \hline
214 < SRE      & $14 \pm 1$ & $15 \pm 2$ & $17 \pm 3$ & $12 \pm 3$ & $15 \pm 3$ & $13 \pm 2$ & $12 \pm 2$ & $16 \pm 2$  \\
211 < \hline
212 < \end{tabular}}
213 < \caption{ \ttdl\ predictions for alternative MC samples. The uncertainties are statistical only.
214 < \label{tab:ttdlalt}}
215 < \end{center}
216 < \end{table}
196 > \begin{figure}[hbt]
197 >  \begin{center}
198 >        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRA.pdf}%
199 >        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRB.pdf}
200 >        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRC.pdf}%
201 >        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRD.pdf}
202 >        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRE.pdf}
203 >        \caption{
204 >          \label{fig:ttllsyst}\protect
205 >          Comparison of the \ttll\ central prediction with those using
206 >          alternative MC samples. The blue band corresponds to the
207 >          total statistical error for all data and MC samples. The
208 >          alternative sample predictions are indicated by the
209 >          datapoints. The uncertainties on the alternative predictions
210 >          correspond to the uncorrelated statistical uncertainty from
211 >          the size of the alternative sample only.  Note the
212 >          suppressed vertical scales.}
213 >      \end{center}
214 >    \end{figure}
215  
216  
217   \begin{table}[!h]
# Line 229 | Line 227 | SRA     & $2$ & $2$ & $5$ & $12$ & $7$ & $
227   \hline
228   SRB      & $6$ & $0$ & $6$ & $5$ & $12$ & $5$ & $6$  \\
229   \hline
230 < SRC      & $10$ & $3$ & $2$ & $12$ & $14$ & $16$ & $4$  \\
231 < \hline
232 < SRD      & $10$ & $6$ & $6$ & $21$ & $15$ & $19$ & $0$  \\
233 < \hline
234 < SRE      & $6$ & $17$ & $15$ & $2$ & $12$ & $17$ & $8$  \\
230 > % SRC    & $10$ & $3$ & $2$ & $12$ & $14$ & $16$ & $4$  \\
231 > % \hline
232 > % SRD    & $10$ & $6$ & $6$ & $21$ & $15$ & $19$ & $0$  \\
233 > % \hline
234 > % SRE    & $6$ & $17$ & $15$ & $2$ & $12$ & $17$ & $8$  \\
235   \hline
236   \end{tabular}}
237 < \caption{ Relative difference in \ttdl\ predictions for alternative MC samples.
237 > \caption{ Relative difference in \ttdl\ predictions for alternative MC
238 >  samples in
239 > the higher statistics regions SRA and SRB.  These differences
240 > are based on the central values of the predictions.  For a fuller
241 > picture
242 > of the situation, including statistical uncertainites, see Fig.~\ref{fig:ttllsyst}.
243   \label{tab:fracdiff}}
244   \end{center}
245   \end{table}
246  
247  
248 < \begin{table}[!h]
249 < \begin{center}
250 < {\footnotesize
248 < \begin{tabular}{l||c|c|c|c|c|c|c}
249 < \hline
250 < $N \sigma$     & Madgraph & Mass Up & Mass Down & Scale Up & Scale Down &
251 < Match Up & Match Down \\
252 < \hline
253 < \hline
254 < SRA      & $0.38$ & $0.42$ & $1.02$ & $2.34$ & $1.58$ & $0.01$ & $0.33$  \\
255 < \hline
256 < SRB      & $1.17$ & $0.07$ & $0.98$ & $0.76$ & $2.29$ & $0.78$ & $1.11$  \\
257 < \hline
258 < SRC      & $1.33$ & $0.37$ & $0.26$ & $1.24$ & $1.82$ & $1.97$ & $0.54$  \\
259 < \hline
260 < SRD      & $0.82$ & $0.46$ & $0.38$ & $1.32$ & $1.27$ & $1.47$ & $0.00$  \\
261 < \hline
262 < SRE      & $0.32$ & $0.75$ & $0.66$ & $0.07$ & $0.66$ & $0.83$ & $0.38$  \\
263 < \hline
264 < \end{tabular}}
265 < \caption{ N $\sigma$ difference in \ttdl\ predictions for alternative MC samples.
266 < \label{tab:nsig}}
267 < \end{center}
268 < \end{table}
248 > In Fig.~\ref{fig:ttllsyst} we compare the alternate MC \ttll\ background predictions
249 > for regions A through E.  We can make the following observations based
250 > on this Figure.
251  
252 + \begin{itemize}
253 + \item In the tighter signal regions we are running out of
254 +  statistics.    
255 + \item Within the limited statistics, there is no evidence that the
256 +  situation changes as we go from signal region A to signal region E.
257 + Therefore, we assess a systematic based on the relatively high
258 + statistics
259 + test in signal region A, and apply the same systematic uncertainty
260 + to all other regions.
261 + \item In order to fully (as opposed as 1$\sigma$) cover the
262 + alternative MC variations in region A we would have to take a
263 + systematic
264 + uncertainty of $\approx 10\%$.  This would be driven by the
265 + scale up/scale down variations, see Table~\ref{tab:fracdiff}.
266 + \end{itemize}
267  
268 < \begin{table}[!h]
268 > \begin{table}[!ht]
269   \begin{center}
270 < \begin{tabular}{l||c|c|c|c}
274 < \hline
275 < Av. $\Delta$ Evt.     & Alt. Gen. & $\Delta$ Mass & $\Delta$ Scale
276 < & $\Delta$ Match \\
277 < \hline
270 > \begin{tabular}{l|c|c}
271   \hline
272 < SRA      & $5.0$ ($1\%$) & $9.6$ ($2\%$) & $56.8$ ($10\%$) & $4.4$ ($1\%$)  \\
272 >            Sample
273 >            &                K3   & K4\\
274   \hline
281 SRB      & $10.4$ ($3\%$) & $9.6$ ($3\%$) & $28.2$ ($9\%$) & $2.8$ ($1\%$)  \\
275   \hline
276 < SRC      & $5.7$ ($5\%$) & $3.1$ ($3\%$) & $14.5$ ($13\%$) & $6.4$ ($6\%$)  \\
277 < \hline
278 < SRD      & $1.9$ ($5\%$) & $0.1$ ($0\%$) & $6.9$ ($18\%$) & $3.6$ ($9\%$)  \\
279 < \hline
280 < SRE      & $0.5$ ($3\%$) & $2.3$ ($16\%$) & $1.0$ ($7\%$) & $1.8$ ($12\%$)  \\
276 > Powheg     & $1.01 \pm 0.03$ & $0.93 \pm 0.04$ \\
277 > Madgraph  & $1.01 \pm 0.04$ & $0.92 \pm 0.04$ \\
278 > Mass Up    & $1.00 \pm 0.04$ & $0.92 \pm 0.04$ \\
279 > Mass Down    & $1.06 \pm 0.04$ & $0.99 \pm 0.05$ \\
280 > Scale Up    & $1.14 \pm 0.04$ & $1.23 \pm 0.06$ \\
281 > Scale Down    & $0.89 \pm 0.03$ & $0.74 \pm 0.03$ \\
282 > Match Up    & $1.02 \pm 0.04$ & $0.97 \pm 0.04$ \\
283 > Match Down    & $1.02 \pm 0.04$ & $0.91 \pm 0.04$ \\
284   \hline
285   \end{tabular}
286 < \caption{ Av. difference in \ttdl\ events for alternative sample pairs.
287 < \label{tab:devt}}
286 > \caption{$\met>100$ GeV: Data/MC scale factors used to account for differences in the
287 >  fraction of events with additional hard jets from radiation in
288 >  \ttll\ events. \label{tab:njetskfactors_met100}}
289   \end{center}
290   \end{table}
291  
292  
293 + However, we have two pieces of information indicating that the
294 + scale up/scale down variations are inconsistent with the data.
295 + These are described below.
296 +
297 + The first piece of information is that the jet multiplicity in the scale
298 + up/scale down sample is the most inconsistent with the data.  This can be shown
299 + in Table~\ref{tab:njetskfactors_met100}, where we tabulate the
300 + $K_3$ and $K_4$ factors of Section~\ref{tab:njetskfactors_met100} for
301 + different \ttbar\ MC samples.  The data/MC disagreement in the $N_{jets}$
302 + distribution
303 + for the scale up/scale down samples is also shown in Fig.~\ref{fig:dileptonnjets_scaleup}
304 + and~\ref{fig:dileptonnjets_scaledw}.  This should be compared with the
305 + equivalent $N_{jets}$ plots for the default Powheg MC, see
306 + Fig.~\ref{fig:dileptonnjets}, which agrees much better with data.
307 +
308   \begin{figure}[hbt]
309    \begin{center}
310 <        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRA.pdf}%
311 <        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRB.pdf}
312 <        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRC.pdf}%
313 <        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRD.pdf}
314 <        \includegraphics[width=0.5\linewidth]{plots/n_dl_comp_SRE.pdf}
315 <        \caption{
316 <          \label{fig:ttllsyst}\protect
305 <          Comparison of the \ttll\ central prediction with those using
306 <          alternative MC samples. The blue band corresponds to the
307 <          total statistical error for all data and MC samples. The
308 <          alternative sample predictions are indicated by the
309 <          datapoints. The uncertainties on the alternative predictions
310 <          correspond to the uncorrelated statistical uncertainty from
311 <          the size of the alternative sample only.
312 <        [TO BE UPDATED WITH THE LATEST SELECTION AND SFS]}
310 >        \includegraphics[width=0.5\linewidth]{plots/njets_all_met50_mueg_scaleup.pdf}
311 >        \includegraphics[width=0.5\linewidth]{plots/njets_all_met50_diel_scaleup.pdf}%
312 >        \includegraphics[width=0.5\linewidth]{plots/njets_all_met50_dimu_scaleup.pdf}
313 >        \caption{
314 >          \label{fig:dileptonnjets_scaleup}%\protect
315 >          SCALE UP: Comparison of the jet multiplicity distribution in data and MC for dilepton events in the \E-\M\
316 >          (top), \E-\E\ (bottom left) and \M-\M\ (bottom right) channels.}  
317        \end{center}
318 <    \end{figure}
318 > \end{figure}
319 >
320 > \begin{figure}[hbt]
321 >  \begin{center}
322 >        \includegraphics[width=0.5\linewidth]{plots/njets_all_met50_mueg_scaledw.pdf}
323 >        \includegraphics[width=0.5\linewidth]{plots/njets_all_met50_diel_scaledw.pdf}%
324 >        \includegraphics[width=0.5\linewidth]{plots/njets_all_met50_dimu_scaledw.pdf}
325 >        \caption{
326 >          \label{fig:dileptonnjets_scaledw}%\protect
327 >          SCALE DOWN: Comparison of the jet multiplicity distribution in data and MC for dilepton events in the \E-\M\
328 >          (top), \E-\E\ (bottom left) and \M-\M\ (bottom right) channels.}  
329 >      \end{center}
330 > \end{figure}
331 >
332 >
333 > \clearpage
334 >
335 > The second piece of information is that we have performed closure
336 > tests in CR5 using the alternative MC samples.  These are exactly
337 > the same tests as the one performed in Section~\ref{sec:CR5} on the
338 > Powheg sample.  As we argued previously, this is a very powerful
339 > test of the background calculation.
340 > The results of this test are summarized in Table~\ref{tab:hugecr5yields}.
341 > Concentrating on the relatively high statistics CR5A region, we see
342 > for all \ttbar\ MC samples except scale up/scale down we obtain
343 > closure within 1$\sigma$.  The scale up/scale down tests closes
344 > worse, only within 2$\sigma$.  This again is evidence that the
345 > scale up/scale down variations are in disagreement with the data.
346 >
347 > \input{hugeCR5Table.tex}
348 >
349 > Based on the two observations above, we argue that the MC
350 > scale up/scale down variations are too extreme.  We feel that
351 > a reasonable choice would be to take one-half of the scale up/scale
352 > down variations in our MC.  This factor of 1/2 would then bring
353 > the discrepancy in the closure test of
354 > Table~\ref{tab:hugecr5yields} for the scale up/scale down variations
355 > from about 2$\sigma$ to about 1$\sigma$.
356 >
357 > Then, going back to Table~\ref{tab:fracdiff}, and reducing the scale
358 > up/scale
359 > down variations by a factor 2, we can see that a systematic
360 > uncertainty
361 > of 6\% would fully cover all of the variations from different MC
362 > samples in SRA and SRB.
363 > {\bf Thus, we take a 6\% systematic uncertainty,  constant as a
364 > function of signal region, as the systematic due to alternative MC
365 > models.}.
366 > Note that this 6\% is also consistent with the level at which we are
367 > able
368 > to test the closure of the method in CR5 for the high statistics
369 > regions
370 > (Table~\ref{tab:hugecr5yields}).
371 >
372 >
373 >
374 >
375 >
376 >
377 > %\begin{table}[!h]
378 > %\begin{center}
379 > %{\footnotesize
380 > %\begin{tabular}{l||c||c|c|c|c|c|c|c}
381 > %\hline
382 > %Sample              & Powheg & Madgraph & Mass Up & Mass Down & Scale
383 > %Up & Scale Down &
384 > %Match Up & Match Down \\
385 > %\hline
386 > %\hline
387 > %SRA     & $579 \pm 10$ & $569 \pm 16$ & $591 \pm 18$ & $610 \pm 22$ & $651 \pm 22$ & $537 \pm 16$ & $578 \pm 18$ & $570 \pm 17$  \\
388 > %\hline
389 > %SRB     & $328 \pm 7$ & $307 \pm 11$ & $329 \pm 13$ & $348 \pm 15$ & $344 \pm 15$ & $287 \pm 10$ & $313 \pm 13$ & $307 \pm 12$  \\
390 > %\hline
391 > %SRC     & $111 \pm 4$ & $99 \pm 5$ & $107 \pm 7$ & $113 \pm 8$ & $124 \pm 8$ & $95 \pm 6$ & $93 \pm 6$ & $106 \pm 6$  \\
392 > %\hline
393 > %SRD     & $39 \pm 2$ & $35 \pm 3$ & $41 \pm 4$ & $41 \pm 5$ & $47 \pm 5$ & $33 \pm 3$ & $31 \pm 3$ & $39 \pm 4$  \\
394 > %\hline
395 > %SRE     & $14 \pm 1$ & $15 \pm 2$ & $17 \pm 3$ & $12 \pm 3$ & $15 \pm 3$ & $13 \pm 2$ & $12 \pm 2$ & $16 \pm 2$  \\
396 > %\hline
397 > %\end{tabular}}
398 > %\caption{ \ttdl\ predictions for alternative MC samples. The uncertainties are statistical only.
399 > %\label{tab:ttdlalt}}
400 > %\end{center}
401 > %\end{table}
402 >
403 >
404 >
405 >
406 > %\begin{table}[!h]
407 > %\begin{center}
408 > %{\footnotesize
409 > %\begin{tabular}{l||c|c|c|c|c|c|c}
410 > %\hline
411 > %$N \sigma$     & Madgraph & Mass Up & Mass Down & Scale Up & Scale Down &
412 > %Match Up & Match Down \\
413 > %\hline
414 > %\hline
415 > %SRA     & $0.38$ & $0.42$ & $1.02$ & $2.34$ & $1.58$ & $0.01$ & $0.33$  \\
416 > %\hline
417 > %SRB     & $1.17$ & $0.07$ & $0.98$ & $0.76$ & $2.29$ & $0.78$ & $1.11$  \\
418 > %\hline
419 > %SRC     & $1.33$ & $0.37$ & $0.26$ & $1.24$ & $1.82$ & $1.97$ & $0.54$  \\
420 > %\hline
421 > %SRD     & $0.82$ & $0.46$ & $0.38$ & $1.32$ & $1.27$ & $1.47$ & $0.00$  \\
422 > %\hline
423 > %SRE     & $0.32$ & $0.75$ & $0.66$ & $0.07$ & $0.66$ & $0.83$ & $0.38$  \\
424 > %\hline
425 > %\end{tabular}}
426 > %\caption{ N $\sigma$ difference in \ttdl\ predictions for alternative MC samples.
427 > %\label{tab:nsig}}
428 > %\end{center}
429 > %\end{table}
430 >
431 >
432 > %\begin{table}[!h]
433 > %\begin{center}
434 > %\begin{tabular}{l||c|c|c|c}
435 > %\hline
436 > %Av. $\Delta$ Evt.     & Alt. Gen. & $\Delta$ Mass & $\Delta$ Scale
437 > %& $\Delta$ Match \\
438 > %\hline
439 > %\hline
440 > %SRA     & $5.0$ ($1\%$) & $9.6$ ($2\%$) & $56.8$ ($10\%$) & $4.4$ ($1\%$)  \\
441 > %\hline
442 > %SRB     & $10.4$ ($3\%$) & $9.6$ ($3\%$) & $28.2$ ($9\%$) & $2.8$ ($1\%$)  \\
443 > %\hline
444 > %SRC     & $5.7$ ($5\%$) & $3.1$ ($3\%$) & $14.5$ ($13\%$) & $6.4$ ($6\%$)  \\
445 > %\hline
446 > %SRD     & $1.9$ ($5\%$) & $0.1$ ($0\%$) & $6.9$ ($18\%$) & $3.6$ ($9\%$)  \\
447 > %\hline
448 > %SRE     & $0.5$ ($3\%$) & $2.3$ ($16\%$) & $1.0$ ($7\%$) & $1.8$ ($12\%$)  \\
449 > %\hline
450 > %\end{tabular}
451 > %\caption{ Av. difference in \ttdl\ events for alternative sample pairs.
452 > %\label{tab:devt}}
453 > %\end{center}
454 > %\end{table}
455 >
456 >
457  
458   \clearpage
459  
# Line 780 | Line 922 | $\mu$ + $\geq$4 jets   &           $>$ 1
922   %      \end{center}
923   %\end{figure}
924  
925 < \subsection{Summary of uncertainties}
926 < \label{sec:bgunc-bottomline}.
925 > % \subsection{Summary of uncertainties}
926 > % \label{sec:bgunc-bottomline}.
927  
928 < THIS NEEDS TO BE WRITTEN
928 > % THIS NEEDS TO BE WRITTEN

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines