Generative.tex

\section*{Discriminative / generative modeling}
Discr. estimate $P(y|x)$, generative $P(y,x)$ \\
Approach (generative): $P(x,y) = P(x|y) \cdot P(y)$
- Estimate prior on labels $P(y)$\\
- Estimate cond. distr. $P(x|y)$ for each class $y$\\
- Pred. using Bayes:
$P(y|x) = \frac{P(y) P(x|y)}{P(x)}$\\
$P(x) = \sum_y P(x,y)$

%\subsection*{Example: Naive Bayes Model}
%cond. ind.:$P(X_1,...,X_d|Y) = \prod_{i=1}^d P(X_i|Y)$

\subsection*{Examples}
MLE for $P(y) = p = \frac{n_+}{n}$\\
MLE for $P(x_i|y) = \mathcal{N}(x_i;\mu_{i,y}, \sigma_{i,y}^2)$:\\
$\hat{\mu}_{i,y} = \frac{1}{n_y} \sum_{x\in D_{x_i|y}} x$\\
$\hat{\sigma}_{i,y}^2 = \frac{1}{n_y} \sum_{x\in D_{x_i|y}} (x-\hat{\mu}_{i,y})^2$\\
MLE for Poi.: $\lambda = \operatorname{avg}(x_i) $\\
$\mathbb{R}^d$: $P(X = x|Y = y) = \prod_{i=1}^dPois(\lambda_y^{(i)},x^{(i)})$


\subsection*{Deriving decision rule}
%In order to predict label y for new point x, use\\
$P(y|x) = \frac{1}{Z} P(y)P(x|y)$, $Z = \sum_y P(y) P(x|y)$\\
$y^* = \underset{y}{\operatorname{amax}} ~ P(y|x) = 
\underset{y}{\operatorname{amax}} ~ P(y) \prod_{i=1}^d P(x_i|y)$

\subsection*{Gaussian Bayes Classifier}
$\hat{P}(x|y) = \mathcal{N}(x ; \hat{\mu}_y, \hat{\Sigma}_y)$\\
$\hat{P}(Y=y) = \hat{p}_y = \frac{n_y}{n}$\\
$\hat{\mu}_{y} = \frac{1}{n_y} \sum_{i:y_i=y} x_i \in \mathbb{R}^d$\\
$\hat{\Sigma}_{y} = \frac{1}{n_y} \sum_{i:y_i=y} (x_i - \hat{\mu}_{y})(x_i-\hat{\mu}_y)^T \in \mathbb{R}^{d \times d}$

\subsection*{Fisher's lin. discrim. analysis (LDA, c=2)}
Assume: $p = 0.5$; $\hat{\Sigma}_- = \hat{\Sigma}_+ = \hat{\Sigma}$\\
discriminant function: 
$f(x) = \operatorname{log} \frac{p}{1-p} + \\
\frac{1}{2}[\operatorname{log} \frac{|\hat{\Sigma}_-|}{|\hat{\Sigma}_+|}
+ \left((x - \hat{\mu}_-)^T \hat{\Sigma}_-^{-1} (x - \hat{\mu}_-)\right) - \\
\left((x - \hat{\mu}_+)^T \hat{\Sigma}_+^{-1} (x - \hat{\mu}_+)\right)]$\\
Predict: $y = \operatorname{sign}(f(x)) = \operatorname{sign} (w^T x + w_0)$\\
$w = \hat{\Sigma}^{-1}(\hat{\mu}_+ - \hat{\mu}_-)$; \\
$w_0 = \frac{1}{2}(\hat{\mu}_-^T\hat{\Sigma}^{-1}\hat{\mu}_- - \hat{\mu}_+^T \hat{\Sigma}^{-1}\hat{\mu}_+)$

\subsection*{Outlier Detection}
$P(x) \leq \tau$

\subsection*{Categorical Naive Bayes Classifier}
MLE for feature distr.:
$\hat{P}(X_i = c|Y = y) = \theta_{c|y}^{(i)}\\
\theta_{c|y}^{(i)} = \frac{Count(X_i = c, Y = y)}{Count(Y=y)}$\\
Prediction: $y^* = \underset{y}{argmax}\hat{P}(y|x)$