\slidecolour \mysection{Bayes Theorem for Distributions} \begin{frame} \frametitle{Chapter \thesection. Single parameter problems} \subsection{Prior and posterior distributions} \begin{block}{{\thesubsection} Prior and posterior distributions} \alert{Data}: $\xvece$ \medskip Model: pdf/pf $f(\xvec|\theta)$ depends on a single parameter $\theta$\\ \qquad$\to$ Likelihood function $f(\xvec|\theta)$\\ \qquad\qquad\qquad considered as a function of $\theta$ for known $\xvec$ \medskip \alert{Prior beliefs}: pdf/pf $\pi(\theta)$ \bigskip Combine using Bayes Theorem \medskip \alert{Posterior beliefs}: pdf/pf $\pi(\theta|\xvec)$ \bigskip Posterior distribution summarises all our current knowledge about the parameter $\theta$ \end{block} \end{frame} \begin{frame} \mysubsubsection{Bayes Theorem}{ The posterior probability (density) function for $\theta$ is \begin{align*} \pi(\theta|\xvec)&=\frac{\pi(\theta)\,f(\xvec|\theta)}{f(\xvec)} \\ \intertext{where\vspace*{-3mm}} f(\xvec)&= \begin{cases} \int_\Theta\pi(\theta)\,f(\xvec|\theta)\,d\theta & \text{if $\theta$ is continuous}, \\ \\ \sum_\Theta\pi(\theta)\,f(\xvec|\theta) & \text{if $\theta$ is discrete}. \end{cases} \end{align*} \pause \medskip As $f(\xvec)$ is not a function of $\theta$, Bayes Theorem can be rewritten as \begin{align*} \pi(\theta|\xvec)&\propto \pi(\theta)\times f(\xvec|\theta) \\ \text{i.e.}~~\text{\bf posterior}&\boldsymbol{\propto}\text{\bf prior}\boldsymbol{\times}\text{\bf likelihood} \end{align*}} \end{frame} \begin{frame} \begin{myexample} \begin{itemize} \item Data \label{ex:botulism} \begin{table}[ht] \begin{center} \begin{tabular}{|lcccccccc|} \hline Year & 1998 & 1999 & 2000 & 2001 & 2002 & 2003 & 2004 & 2005 \\ Cases & 2 & 0 & 0 & 0 & 1 & 0 & 2 & 1\\ \hline \end{tabular} \caption{Number of cases of foodbourne botulism in England and Wales, 1998--2005} \label{tab:botulism} \end{center} \end{table} \item Assume that cases occur at random at a constant rate~$\theta$ in time \item This means the data are from a Poisson process and so are a random sample from a Poisson distribution with rate $\theta$ \item An expert was consulted to construct a suitable prior distribution for the rate ~$\theta$ \end{itemize} \end{myexample} \end{frame} \begin{frame} \mysubsubsection{Example 1.1}{ %\frametitle{Example 1.1} \begin{itemize} \item [(a)] What might be an appropriate probability model for the rate $\theta$? Why is this termed the \textit{prior distribution}? \vspace{0.1cm} \item [(b)] Suppose the epidemiologist tells you that she expects around two cases of foodbourne botulism per year in England and Wales. After further discussions with the epidemiologist, you are also able to elicit that $SD(\theta) \approx \sqrt{2}$. Using your answer to part (a) find your full prior distribution for $\theta$, and write down its density. \vspace{0.1cm} \item [(c)] Determine the posterior distribution for ~$\theta$. \end{itemize} } \end{frame} \begin{frame} \begin{soln}{\tiny\color{white}Solution...} \end{soln} \end{frame} \begin{frame} \frametitle{Summary} \begin{block}{} \begin{itemize} \item Model: $X_i|\theta\sim Po(\theta)$, $i=1,2,\ldots,8$ (independent) \item Prior: $\theta\sim Ga(2,1)$ \item Data: in Table above \item Posterior: $\theta|\xvec\sim Ga(8,9)$ \end{itemize} \vspace*{-7mm} \begin{figure}[h!] \begin{center} \includegraphics[width=0.6\textwidth]{botulism} \vspace*{-3mm} %\includegraphics[angle=270,width=7cm]{gammaplot2.eps} \caption{Prior (dashed) and posterior (solid) densities for $\theta$} \label{fig:botulismpost} \end{center} \end{figure} \end{block} \end{frame} \begin{frame} \begin{block}{} \begin{table}[ht] \begin{center} \begin{tabular}{|l|c|c|c|} \cline{2-4} \multicolumn{1}{c|}{~}& Prior & Likelihood & Posterior \\ \hline $Mode(\theta)$ & 1.00 & 0.75 & 0.78 \\ $E(\theta)$ & 2.00 & -- & 0.89 \\ $SD(\theta)$ & 1.41 & -- & 0.31 \\ \hline \end{tabular} \caption{Changes in beliefs about $\theta$} \label{tab:botulismsum} \end{center} \end{table} \begin{itemize} \item Likelihood mode $<$ prior mode $\to$ posterior mode moves in direction of likelihood mode $\to$ posterior mode $<$ prior mode \item Reduction in variability from the prior to the posterior \end{itemize} \end{block} \end{frame} \begin{frame} \begin{myexample} We now consider the general case (of Example~{\thesection}.\ref{ex:botulism}). \label{ex:poisgam} Suppose $X_i|\theta\sim Po(\theta)$, $i=1,2,\ldots,n$ (independent) and our prior beliefs about $\theta$ are summarised by a $Ga(g,h)$ distribution (with $g$ and $h$ known), with density \begin{equation} \label{eq:poprior} \pi(\theta)=\frac{h^g\,\theta^{g-1}e^{-h\theta}}{\Gamma(g)}, \quad\theta>0. \end{equation} Determine the posterior distribution for $\theta$. \end{myexample} \begin{soln}{\tiny\color{white}\begin{align}\label{eq:polik}\\\label{eq:popost}\end{align}}\end{soln} \end{frame} \begin{frame} \frametitle{Summary} \begin{block}{} \begin{itemize} \item Model: $X_i|\theta\sim Po(\theta)$, $i=1,2,\ldots,n$ (independent) \item Prior: $\theta\sim Ga(g,h)$ \item Data: observe $\xvec$ \item Posterior: $\theta|\xvec\sim Ga(g+n\bar x,h+n)$ \item Taking $g\ge 1$ \end{itemize} \begin{table}[ht] \begin{center} \begin{tabular}{|l|c|c|c|} \cline{2-4} \multicolumn{1}{c|}{~}& Prior & Likelihood & Posterior \\ \multicolumn{1}{c|}{~}& $\eqref{eq:poprior}$ & $\eqref{eq:polik}$ & $\eqref{eq:popost}$ \\ \hline $Mode(\theta)$ & $(g-1)/h$ & $\bar x$ & $(g+n\bar x-1)/(h+n)$ \\ $E(\theta)$ & $g/h$ & -- & $(g+n\bar x)/(h+n)$ \\ $SD(\theta)$ & $\sqrt{g}/h$ & -- & $\sqrt{g+n\bar x}/(h+n)$ \\ \hline \end{tabular} \caption{Changes in beliefs about $\theta$} \label{tab:gam} \end{center} \end{table} \end{block} \end{frame} \begin{frame} \begin{block}{Comments} \begin{itemize} \item Posterior mean is greater than the prior mean if and only if the likelihood mode is greater than the prior mean, that is, \[ E(\theta|\xvec)>E(\theta)\quad\iff\quad Mode_\theta\{f(\xvec|\theta)\}>E(\theta) \] \item Standard deviation of the posterior distribution is smaller than that of the prior distribution if and only if the sample mean is not too large, that is \[ SD(\theta|\xvec)E(\mu)\quad\iff\quad Mode_\mu\{f(\xvec|\mu)\}>E(\mu) \] \item Standard deviation of the posterior distribution is smaller than that of the prior distribution, that is \[ SD(\mu|\xvec)0$. Therefore we need a family with density $f(\theta|\vec{a})$ and parameters $\vec{a}$ such that \begin{align*} f(\theta|\vec{A})&\propto f(\theta|\vec{a})\times \theta^{n\bar x} e^{-n\theta},\quad\theta>0\\ \noalign{\medskip} &\Longrightarrow\quad f(\theta|\vec{a})\propto\theta^{a_1}e^{-a_2\theta},\quad\theta>0 \end{align*} that is, the Gamma family of distributions \end{block} \end{frame} \begin{frame} \begin{block}{{\thesubsection}.3 Vague prior knowledge/prior ignorance} \begin{itemize} \item Have very little or no prior information about $\theta$ \pause \item Still must choose a prior distribution \pause \item Sensible to choose a prior distribution which is not concentrated about any particular value, that is, one with a very large variance \pause \item Most of the information about $\theta$ will be passed through to the posterior distribution via the data, and so we have $\pi(\theta|\xvec)\sim f(\xvec|\theta)$ \pause \item Vague prior: Represent vague prior knowledge by using a prior distribution which is conjugate to the model for $\xvec$ and which has as large a variance as possible \item Prior ignorance: Uniform prior? Jeffreys prior? \begin{equation} \label{eq:fisher} I(\theta)=E_{\Xvec|\theta}\left[- \frac{\partial^2}{\partial\theta^{2}}\, \log f(\Xvec|\theta)\right], \end{equation} and then: \begin{equation} \pi(\theta)\propto \sqrt{I(\theta)}. \end{equation} \end{itemize} \end{block} \end{frame} \begin{frame} \begin{myexample} In constructing the prior in Example 1.1, did we assume (i) substantial prior knowledge; (ii) limited prior knowledge; or (iii) prior ignorance? \end{myexample} \begin{soln}{}\end{soln} \end{frame} \begin{frame} \begin{myexample} Suppose the parameter $\theta$ represents a probability. What might be the simplest prior to use here, if we are prior ignorant? \end{myexample} \begin{soln}{}\end{soln} \end{frame} \begin{frame} \begin{myexample} Suppose we have a random sample from a $N(\mu,1/\tau)$ distribution (with $\tau$ known). Determine the posterior distribution assuming a vague prior for $\mu$. \end{myexample} \begin{soln}{}\end{soln} \end{frame} \begin{frame} \begin{myexample} Suppose we have a random sample from an Poisson distribution, that is, $X_i|\theta\sim Po(\theta)$, $i=1,2,\ldots,n$ (independent). Determine the posterior distribution assuming a vague prior for $\theta$. \end{myexample} \begin{block}{Solution} \begin{itemize} \item The conjugate prior distribution is a Gamma distribution \item The $Ga(g,h)$ distribution has mean $m=g/h$ and variance $v=g/h^2$ \item Rearranging gives $g=m^2/v$ and $h=m/v$ \item Clearly $g\to 0$ and $h\to 0$ as $v\to\infty$ (for fixed~$m$) \item We have seen that, for this model, using a $Ga(g,h)$ prior distribution results in a $Ga(g+n\bar x,h+n)$ posterior distribution \item Therefore, taking a vague prior distribution will give a $Ga(n\bar x,n)$ posterior distribution \end{itemize} Note that the posterior mean is $\bar x$ (the likelihood mode) and that the posterior variance $\bar x/n\to 0$ and $n\to\infty$. \end{block} \end{frame} \begin{frame} \begin{myexample} Suppose we have a random sample from an exponential distribution, that \label{ex:38} is, $X_i|\theta\sim \textrm{\it Exp}(\theta)$, $i=1,2,\ldots,n$ (independent). Determine the Jeffreys prior for this model. \end{myexample} \begin{soln}{}\end{soln} \end{frame} \slidecolour \begin{frame} \frametitle{What have we done so far?} \textbf{1. $\text{\bf Posterior} \boldsymbol{\propto} \text{\bf Prior} \boldsymbol{\times} \text{\bf Likelihood}$} \vspace{0.15cm} \begin{itemize} \item \textbf{Example:} \textcolor{blue}{\bf Poisson distribution} with gamma prior (food poisoning, and generic/algebraic example) \pause \item \textbf{Example:} \textcolor{blue}{\bf Normal distribution} (known precision) with Normal prior for the mean (generic/algebraic example, and Cavendish experiment). \textcolor{red}{(Memorise the results from the algebraic example!)} \pause \item \textbf{Example:} \textcolor{blue}{\bf Binomial distribution} with beta prior \textcolor{red}{(homework)} \pause \item \textbf{Example:} \textcolor{blue}{\bf Exponential distribution} with Normal prior \textcolor{red}{(homework)} \pause \item Make sure you're familiar with simple functions in \texttt{R} for computing probabilities (e.g. \textcolor{blue}{\texttt{pnorm}}), and make sure you can sketch/plot distributions! \end{itemize} \end{frame} \begin{frame} \frametitle{What have we done so far?} \textbf{2. Different levels of prior information} \begin{itemize} \item \textcolor{blue}{\bf Substantial} prior information: $\pi(\theta|\boldsymbol{x}) \sim \pi(\theta)$, but often difficult to construct \pause \item \textcolor{blue}{\bf Limited} prior information: Use conjugate prior with parameters determined by the information provided by the expert \pause \item \textcolor{blue}{\bf Vague} prior: Use conjugate prior with maximised variance \pause \item \textcolor{blue}{\bf Ignorance} prior: Use a Jeffreys prior, or perhaps a $U(0,1)$? \pause \item For vague/ignorance priors, $\pi(\theta|\boldsymbol{x}) \sim f(\boldsymbol{x}|\theta)$ \pause \item Lots of \textbf{Examples}: Examples 1.5--1.9 in notes \pause \end{itemize} \vspace{0.5cm} \textbf{3. Posterior distribution when the sample is very large} \vspace{0.5cm} \textbf{4. Bayesian inference -- confidence intervals and prediction} \end{frame} \slidecolour \begin{frame} \subsection{Asymptotic Posterior Distribution} \frametitle{{\thesubsection} Asymptotic Posterior Distribution} \begin{block}{Background} \begin{itemize} \item There are many asymptotic results in Statistics \item The \textbf{Central Limit Theorem} is a statement about the asymptotic distribution of $\bar X_n$ as the sample size $n\to\infty$, where the $X_i$ are i.i.d. with known mean $\mu$ and known (finite) variance $\sigma^2$ \item Under different distributions of $X_i$, $\bar X_n$ has the same moments, $E(\bar X_n)=\mu$ and $Var(\bar X_n)=\sigma^2/n$, but its distribution varies \item The CLT says that as $n\to\infty$, regardless of the distribution of $X_i$, \[ \frac{\sqrt{n}(\bar X_n-\mu)}{\sigma}\stackrel{\cal D} \longrightarrow N(0,1) \] \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Theorem} Suppose we have a statistical model $f(\xvec|\theta)$ for data $\xvece$, together with a prior distribution $\pi(\theta)$ for $\theta$. Then \[ \sqrt{J(\thetahat)}\,(\theta-\thetahat)|\xvec \stackrel{\cal D} \longrightarrow N(0,1) \quad\quad\text{as }n\to\infty, \] where $\thetahat$ is the likelihood mode and $J(\theta)$ is the observed information \[ J(\theta)=-\frac{\partial^2}{\partial\theta^2}\, \log f(\xvec|\theta) \] \end{block} \begin{block}{Usage} In large samples, \[ \theta|\xvec\sim N\left(\hat\theta,J(\hat\theta)^{-1} \right), \quad\quad\text{approximately.} \] \end{block} \begin{block}{} \begin{itemize} \item In large samples ($n$ large), how the prior distribution is specified does not matter. \end{itemize} \end{block} \end{frame} \begin{frame} \vspace*{-3mm} \begin{myexample} Suppose we have a random sample from a $N(\mu,1/\tau)$ distribution (with $\tau$~known). Determine the asymptotic posterior distribution for~$\mu$. Recall that {\small\[ f(\xvec|\mu)= \left(\frac{\tau}{2\pi}\right)^{n/2} \exp\left\{-\frac{\tau}{2}\sum_{i=1}^n (x_i-\mu)^2\right\}, \]} and therefore {\small \begin{align*} \log\,&f(\xvec|\mu)=\frac{n}{2}\log\tau-\frac{n}{2}\log(2\pi) -\frac{\tau}{2}\sum_{i=1}^n (x_i-\mu)^2 \\ &\Rightarrow\quad \frac{\partial}{\partial\mu} \log f(\xvec|\mu)=-\frac{\tau}{2}\times\sum_{i=1}^n -2(x_i-\mu) =\tau\sum_{i=1}^n (x_i-\mu) =n\tau(\bar x-\mu)\\ &\Rightarrow\quad \frac{\partial^2}{\partial\mu^2} \log f(\xvec|\mu)=-n\tau \qquad\Rightarrow\quad J(\mu)=-\frac{\partial^2}{\partial\mu^2} \log f(\xvec|\mu)=n\tau. \end{align*} } \end{myexample} \begin{soln}{}\end{soln} \begin{block}{} Here the asymptotic posterior distribution is the same as the posterior distribution under vague prior knowledge \end{block} \end{frame} \slidecolour \subsection{Bayesian inference} \begin{frame} \frametitle{{\thesubsection} Bayesian inference} \begin{block}{} \begin{itemize} \item The posterior distribution $\pi(\theta|\xvec)$ summarises all our information about $\theta$ to date \item It can answer the questions: How to estimate the value of $\theta$, and what is the uncertainty of the estimator? \end{itemize} \end{block} \begin{block}{{\thesubsection}.1 Estimation} \end{block} \begin{block}{Point estimates} Many useful summaries, such as \begin{itemize} \item the mean: $E(\theta|\xvec)$ \item the mode: $Mode(\theta|\xvec)$ \item the median: $Median(\theta|\xvec)$ \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Interval Estimates} \begin{itemize} \item A more useful summary of the posterior distribution is one which also reflects its variation \item A $100(1-\alpha)\%$ \textit{Bayesian confidence interval} for $\theta$ is any region $C_\alpha$ that satisfies $Pr(\theta\in C_\alpha|\xvec)=1-\alpha$ \item If $\theta$ is continuous with posterior density $\pi(\theta|\xvec)$ then \[ \int_{C_\alpha} \pi(\theta|\xvec)\,d\theta = 1-\alpha \] \pause \item The usual correction is made for discrete $\theta$, that is, we take the largest region $C_\alpha$ such that $Pr(\theta\in C_\alpha|\xvec)\leq 1-\alpha$ \item Bayesian confidence intervals are sometimes called \textit{credible regions} or \textit{plausible regions} \item Clearly these intervals are not unique, since there will be many intervals with the correct probability coverage for a given posterior distribution \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Highest density intervals} \begin{itemize} \item A $100(1-\alpha)\%$ {\it highest density interval} (HDI) for $\theta$ is the region \[ C_\alpha=\{\theta:~\pi(\theta|\xvec)\geq\gamma\} \] where $\gamma$ is chosen so that $Pr(\theta\in C_\alpha|\xvec)=1-\alpha$ \item It is a $100(1-\alpha)\%$ Bayesian confidence interval but only includes the most likely values of $\theta$ \item This region is sometimes called a \textit{most plausible Bayesian confidence interval} \item If the posterior distribution has many modes then it is possible that the HDI will be the union of several disjoint regions \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Highest density intervals} \begin{itemize} \item If the posterior distribution is \alert{unimodal} (has one mode) and symmetric about its mean then the \alert{HDI is an equi-tailed interval}, that is, takes the form $C_\alpha=(a,b)$, where \vspace*{-3mm} \[ Pr(\thetab|\xvec)=\alpha/2 \] \end{itemize} \end{block} \begin{figure}[ht] \begin{center} \includegraphics[width=0.6\textwidth]{hdi} \vspace*{-4mm} \caption{HDI for $\theta$} %\label{fig:hdi} \end{center} \end{figure} \end{frame} \begin{frame} \begin{myexample} Suppose we have a random sample $\xvece$ from a $N(\mu,1/\tau)$ distribution (where $\tau$ is known). We have seen that, assuming vague prior knowledge, the posterior distribution is $\mu|\xvec\sim N\{\bar x,1/(n\tau)\}$. Determine the $100(1-\alpha)\%$ HDI for~$\mu$. \end{myexample} \begin{soln}{}\end{soln} \begin{block}{Comment} Note that this interval is numerically identical to the 95\% frequentist confidence interval for the (population) mean of a normal random sample with known variance. However, the interpretation is very different. \end{block} \end{frame} \begin{frame} \begin{block}{Interpretation of confidence intervals} \begin{itemize} \item[] $C_B$ is a 95\% Bayesian confidence interval for $\theta$ \item[] $C_F$ is a 95\% frequentist confidence interval for $\theta$ \end{itemize} These intervals do not have the same interpretation: \begin{itemize} \item the probability that $C_B$ contains $\theta$ is 0.95 \item the probability that $C_F$ contains $\theta$ is either 0 or 1 \item the interval $C_F$ covers the true value $\theta$ on 95\% of occasions --- in repeated applications of the formula \end{itemize} \end{block} \end{frame} \begin{frame} \vspace*{-3mm} \begin{myexample} Recall Example~{\thesection}.\ref{ex:botulism} on the number of cases of foodbourne botulism in England and Wales. The data were modelled as a random sample from a Poisson distribution with mean~$\theta$. Using a $Ga(2,1)$ prior distribution, we found the posterior distribution to be $\theta|\xvec\sim Ga(8,9)$, with density \vspace*{-4mm} \begin{figure}[ht] \begin{center} \includegraphics[width=0.45\textwidth]{botulism2} \vspace*{-4mm} \caption{Posterior density for $\theta$} \label{fig:botulismpost2} \end{center} \end{figure} \vspace*{-4mm} Determine the 95\% HDI for~$\theta$. \end{myexample} \vspace*{-2mm} \begin{soln}{}\end{soln} \end{frame} \begin{frame} \begin{block}{Simple way to calculate the HDI} \begin{itemize} \item Use the \R{R} function \R{hdiGamma} in the package \R{nclbayes} \item It calculates the HDI for any Gamma distribution \item Here we use\\ \R{library(nclbayes)}\\ \R{hdiGamma(p=0.95,a=8,b=9)} \bigskip \item The package also has functions \begin{itemize} \item \R{hdiBeta} for the Beta distribution \item \R{hdiInvchi} for the Inv-Chi distribution (introduced in Chapter~\alert{3}) \end{itemize} \end{itemize} \end{block} \end{frame} \slidecolour \begin{frame} \frametitle{{\thesubsection}.2 Prediction} \begin{block}{} \begin{itemize} \item Much of statistical inference (both Frequentist and Bayesian) is aimed towards making statements about a parameter $\theta$ \item Often the inferences are used as a yardstick for similar future experiments \item For example, we may want to predict the outcome when the experiment is performed again \end{itemize} \end{block} \begin{block}{Predicting the future} \begin{itemize} \item There will be uncertainty about the future outcome of an experiment \item Suppose this future outcome $Y$ is described by a probability (density) function $f(y|\theta)$ \item If $\theta$ were known, say $\theta_0$, then any prediction can do no better than one based on $f(y|\theta=\theta_0)$ \item What if $\theta$ is unknown? \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Frequentist solution} \begin{itemize} \item Get estimate $\thetahat$ and use $f(y|\theta=\thetahat)$ but this ignores uncertainty on $\thetahat$ \item Better: use $E_{\thetahat}\{f(y|\theta=\thetahat)\}$ to average over uncertainty on $\thetahat$ \end{itemize} \end{block} \begin{block}{Bayesian solution} \begin{itemize} \item Use the \textit{predictive distribution}, with density \[ f(y|\xvec)=\int_\Theta f(y|\theta)\,\pi(\theta|\xvec)\,d\theta \] when $\theta$ is a continuous quantity \item Notice this could be rewritten as \[ f(y|\xvec)=E_{\theta|\xvec}\{f(y|\theta)\} \] \item Uses $f(y|\theta)$ but weights each $\theta$ by our posterior beliefs \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Prediction interval} \begin{itemize} \item Useful range of plausible values for the outcome of a future experiment \item Similar to a HDI interval \item A $100(1-\alpha)\%$ {\it prediction interval} for $Y$ is the region \[ C_\alpha=\{y:~f(y|\xvec)\geq\gamma\} \] where $\gamma$ is chosen so that $Pr(Y\in C_\alpha|\xvec)=1-\alpha$ \end{itemize} \end{block} \end{frame} \begin{frame} \begin{myexample} Recall Example~{\thesection}.\ref{ex:botulism} on the number of cases of \label{ex:botulism2} foodbourne botulism in England and Wales. \vspace{0.2cm} The data for 1998--2005 were modelled as a random sample from a Poisson distribution with mean~$\theta$. \vspace{0.2cm} Using a $Ga(2,1)$ prior distribution, we found the posterior distribution to be $\theta|\xvec\sim Ga(8,9)$. \vspace{0.2cm} Determine the predictive distribution for the number of cases for the following year (2006). \end{myexample} \begin{soln}{}\end{soln} \end{frame} \begin{frame} \begin{block}{Comments} \begin{itemize} \item This predictive probability function is related to that of a negative binomial distribution \pause \item If $Z\sim NegBin(r,p)$ then \[ Pr(Z=z)=\binom{z-1}{r-1}p^r(1-p)^{z-r},\quad z=r,r+1,\ldots \] and so $W=Z-r$ has probability function \[ Pr(W=w)=Pr(Z=w+r)=\binom{w+r-1}{r-1}p^r(1-p)^w,\quad w=0,1,\ldots \] \pause \item This is the same probability function as our predictive probability function, with $r=8$ and $p=0.9$ \item Therefore $Y|\xvec\sim NegBin(8,0.9)-8$ \pause \item Note that, unfortunately, \R{R} also calls the distribution of~$W$ a negative binomial distribution with parameters~$r$ and~$p$: \R{dnbinom(y,r,p)} \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{} \begin{itemize} \item To distinguish between this distribution and the $NegBin(r,p)$ distribution used above, we shall denote the distribution of~$W$ as a $NegBin_{\texttt{R}}(r,p)$ distribution -- it has mean $r(1-p)/p$ and variance $r(1-p)/p^2$ \pause \item Thus $Y|\xvec\sim NegBin_{\texttt{R}}(8,0.9)$ \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Comparison between predictive and naive predictive distributions} \begin{itemize} \item Can compare the predictive distribution $Y|\xvec$ with a naive predictive $Y|\theta=\hat\theta\sim Po(0.75)$ where \[ f(y|\theta=\thetahat)=\frac{0.75^y\,e^{-0.75}}{y!},\quad y=0,1,\ldots~. \] \item Probability functions: \begin{center} \begin{tabular}{|r|c|c|} \cline{2-3} \multicolumn{1}{c|}{~}& correct & naive \\ \cline{1-1} $y$ & $f(y|\xvec)$ & $f(y|\theta=\thetahat)$ \\ \textcolor{red}{\texttt{y = 0:5}} &\textcolor{red}{\texttt{dnbinom(y,8,0.9)}} &\textcolor{red}{\texttt{dpois(y,0.75)}} \\ \hline 0 & 0.430 & 0.472 \\ 1 & 0.344 & 0.354 \\ 2 & 0.155 & 0.133 \\ 3 & 0.052 & 0.033 \\ 4 & 0.014 & 0.006 \\ 5 & 0.003 & 0.001 \\ $\geq 6$ & 0.005 & 0.002 \\ \hline \end{tabular} \end{center} \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{} \begin{itemize} \item The naive predictive distribution is a predictive distribution which uses a degenerate posterior distribution $\pi^*(\theta|\xvec)$ \pause \item Here $Pr_{\pi^*}(\theta=0.75|\xvec)=1$ and standard deviation $SD_{\pi^*}(\theta|\xvec)=0$ \pause \item The correct posterior standard deviation of $\theta$ is $SD_\pi(\theta|\xvec)=\sqrt{8}/9=0.314$ \pause \item Using a degenerate posterior distribution results in the naive predictive distribution having too small a standard deviation: \[ SD(Y|\xvec)= \begin{cases} 0.994 & \text{using the correct }\pi(\theta|\xvec) \\ 0.866 & \text{using the naive }\pi^*(\theta|\xvec), \\ \end{cases} \] these values being calculated from $NegBin_{\texttt{R}}(8,0.9)$ and $Po(0.75)$ distributions \pause \item $\{0,1,2\}$ is a 92.9\% prediction set/interval \pause \item $\{0,1,2\}$ is a 95.9\% prediction set/interval using the the more ``optimistic'' naive predictive distribution \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Predictive distribution (general case)} \begin{itemize} \item Generally requires calculation of a non-trivial integral (or sum) \[ \alert<4>{f(y|\xvec)=\int_\Theta f(y|\theta)\,\pi(\theta|\xvec)\,d\theta} \] \pause \item Easier method available when using a conjugate prior distribution \item Suppose $\theta$ is a continuous quantity and \alert<2>{$\Xvec$ and $Y$ are independent given $\theta$} \item Using Bayes Theorem, the posterior distribution for $\theta$ given $\xvec$ and $y$ is \begin{align*} \pi(\theta|\xvec,y)&=\frac{\pi(\theta)\alert<2>{f(\xvec,y|\theta)}}{f(\xvec,y)}\\ &=\frac{\alert<3>{\pi(\theta)}\alert<2-3>{f(\xvec|\theta)}\alert<2>{f(y|\theta)}}{\alert<3>{f(\xvec)}f(y|\xvec)} \qquad\text{since $\Xvec$ and $Y$ are indep given $\theta$}\\ &=\frac{\alert<3>{\pi(\theta|\xvec)}\,f(y|\theta)}{\alert<4>{f(y|\xvec)}}. \end{align*} \vspace*{-2mm} Rearranging, we obtain $\ldots$ \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{Candidate's formula} \begin{itemize} \item The predictive p(d)f is \[ f(y|\xvec)= \frac{f(y|\theta)\pi(\theta|\xvec)}{\pi(\theta|\xvec,y)} \] \item The RHS looks as if it depends on $\theta$ but it doesn't: all terms in $\theta$ cancel \item For this formula to be useful, we have to be able to work out $\theta|\xvec$ and $\theta|\xvec,y$ fairly easily \item This is the case when using conjugate priors \end{itemize} \end{block} \end{frame} \begin{frame} \begin{myexample} Rework Example~{\thesection}.\ref{ex:botulism2} using Candidate's formula to determine the number of cases in 2006. \end{myexample} \begin{soln}{}\end{soln} \end{frame} \slidecolour \begin{frame} \subsection{Sequential Bayesian Updating} \label{sec:seq} \frametitle{1.5 Sequential Bayesian Updating} \begin{block}{} \begin{itemize} \item Last year, and so far this year: Bayes Theorem to update beliefs about $\theta$ after observing a single sample $\xvec$ \pause \item Real-world: New samples of data coming online continuously! \pause \item First sample $\xvec^{(1)}$ + $\pi(\theta)\longrightarrow\pi(\theta|\xvec^{(1)})$ \pause \item New sample $\xvec^{(2)}$? \begin{itemize} \item [--] Use $\pi(\theta|\xvec^{(1)})$ as the ``new prior'' \item [--] Update once again to obtain the ``new posterior'' $\pi(\theta|\xvec^{(1)}, \xvec^{(2)})$ \end{itemize} \end{itemize} \end{block} \end{frame} \begin{frame} \frametitle{1.5 Sequential Bayesian Updating} \begin{block}{} In other words, we might replace the usual mantra \[ \text{Posterior} \propto \text{Prior} \times \text{Likelihood} \] with \pause \[ \text{Revised} \propto \text{Current} \times \text{New likelihood}, \] with the ability to update our likelihood function with each new sample of data. Why does this work? \end{block} \end{frame} \begin{frame} \frametitle{1.5 Sequential Bayesian Updating} \begin{block}{} Suppose we have data assumed drawn from a probability distribution with parameter $\theta$ \pause \begin{itemize} \item [1.] Before we make any observations at all, our prior beliefs about $\theta$ are represented by $\pi(\theta)$. We might refer to this as our ``original prior'' \pause \item [2.]Now we observe a sample $\xvec = (x_1, x_2, \ldots, x_n)^{T}$; our posterior is then \[ \pi(\theta|\xvec) \propto \pi(\theta) \times f(\xvec|\theta) = \pi(\theta) \times \prod_{i=1}^{n}f(x_{i}|\theta), \] assuming our observations are a random sample \end{itemize} \end{block} \end{frame} \begin{frame} \frametitle{1.5 Sequential Bayesian Updating} \begin{block}{} \begin{itemize} \item [3.] Now suppose we make a new observation $x_{n+1}$ assumed from the same probability distribution as $\xvec$ \pause \begin{itemize} \item [--] Beforehand our \textit{current} beliefs about $\theta$ are represented by $\pi(\theta|\xvec)$; re-label this $\pi_{n}(\theta)$, our ``new prior'' \pause \item [--] Assume the \textit{new likelihood} has the same form as the likelihood for $\xvec$ \pause \item [--] Then \begin{eqnarray*} \text{Revised} &\propto& \text{Current} \times \text{New likelihood} \\ \pause \pi_{n+1}(\theta|x_{n+1}) &\propto& \pi_{n}(\theta) \times f(x_{n+1}|\theta) \\ \pause &\propto& \pi(\theta|\xvec) \times f(x_{n+1}|\theta) \\ \pause &\propto& \left\{\pi(\theta) \times \prod_{i=1}^{n}f(x_{i}|\theta)\right\} \times f(x_{n+1}|\theta) \\ \pause &\propto& \pi(\theta) \times \prod_{i=1}^{n+1}(x_{i}|\theta) \\ \pause &\propto& \text{original prior} \times \text{full likelihood} \end{eqnarray*} \end{itemize} \end{itemize} \end{block} \end{frame} \begin{frame} \begin{myexample} Table 1.6 provides an update to table~\ref{tab:botulism}, giving the number of cases of foodbourne botulism in England and Wales between 2006 and 2010. Use your solution to Example 1.1, and sequential Bayesian updating, to obtain your new posterior distribution for $\theta$. \begin{table}[ht] \begin{center} \begin{tabular}{|lccccc|} \hline Year & 2006 &2007 & 2008 & 2009 & 2010 \\ Cases & 1 & 0 & 0 & 1 & 1 \\ \hline \end{tabular} \caption{Number of cases of foodbourne botulism in England and Wales, 2006--2010} \label{tab:botulism2} \end{center} \end{table} \end{myexample} \begin{soln}{}\end{soln} \end{frame} \begin{frame} \begin{myexample} Devise a generic sequential Bayesian updating plan for successive random samples from a Poisson distribution, assuming a conjugate prior specification. \end{myexample} \begin{soln}{}\end{soln} \end{frame} \slidecolour \begin{frame} \subsection{Learning objectives} \frametitle{1.6 Learning objectives} \begin{block}{} By the end of this chapter, you should be able to: \begin{itemize} \item[1.] Determine the likelihood function using a random sample from \textbf{any} distribution \item[2.] Combine this likelihood function with \textbf{any} prior distribution to obtain the posterior distribution \item[3.] Name the posterior distribution if it is a ``standard'' distribution listed in these notes or on the exam paper -- this list may well include distributions that are standard within the subject but which you have not met before. If the posterior is not a ``standard'' distribution then it is okay just to give its density (or probability function) up to a constant. \item[4.] Do all the above for a particular data set or for a general case with random sample $x_1,\ldots,x_n$ \end{itemize} \end{block} \end{frame} \begin{frame} \begin{block}{} \begin{itemize} \item[5.] Describe the different levels of prior information; determine and use conjugate priors and vague priors \item[6.] Determine the asymptotic posterior distribution \item[7.] Determine the predictive distribution, particularly when having a random sample from any distribution and a conjugate prior via Candidate's formula \item[8.] Describe and calculate the confidence intervals, HDIs and prediction intervals \item[9.] Operate sequential Bayesian updating schemes \end{itemize} \end{block} \end{frame}