\documentclass[english, a4paper, 12pt, reqno]{amsart}% article ams
\usepackage[english]{babel}% account for English typographical particulars. Replaces francais and frenchb
\usepackage[T1]{fontenc}% T1 character encoding, new LaTeX standard. Avoids [OT1] error but switches to Bitmap
\usepackage{lmodern}% Latin Modern font. Switches back from Bitmap to vectorial
\usepackage[left=1.8cm,right=2.2cm,top=2.5cm,bottom=2.5cm]{geometry}% page margins
\def\emailaddrname{Email address }
\def\urladdrname{Website }
\usepackage{amssymb}% for mathematical symbols
\usepackage{pst-all}% for drawings
\usepackage{titlesec}% title formatting
\titleformat{\section}[block]{\vspace{\baselineskip}\large\scshape}{\thesection}{.7cm}{}[\hrule\vspace{.75\baselineskip}]
\titleformat{\subsection}[block]{\vspace{.75\baselineskip}\bfseries}{\thesubsection}{.5cm}{}[]
\renewcommand{\numberline}[1]{#1~~}% ToC numbering issue
\usepackage{fancyhdr}% page numbering
\fancyhf{}% clear head and foot
\pagestyle{fancy}
\fancyhead[LE,RO]{\bfseries\thepage}% page number positions
\renewcommand{\headrulewidth}{0pt}% no header rule
\setlength{\headheight}{16pt}% fixes \headheight too small error
\usepackage{tcolorbox}% frame text
\tcbuselibrary{skins,breakable}
\newtcolorbox{defbox}{colframe=black,colback=white,boxrule=0.4pt}
\newtcolorbox{thbox}{enhanced jigsaw,drop shadow,colframe=black,colback=white,boxrule=0.4pt}
\usepackage{hyperref}
\hypersetup{
pdftoolbar=true,% show Acrobat toolbar
pdfmenubar=true,% show Acrobat menu
pdftitle={Central Limit Theorem},% title
pdfauthor={Olivier Castéra},% author
pdfstartview= FitH,% page fits full width
colorlinks=true,% false: boxed links; true: colored links
linkcolor=blue,% internal link color
}
\theoremstyle{plain}
\newtheorem{theoreme}{Theorem}[section]
\theoremstyle{definition}
\newtheorem{definition}{Definition}[section]
\theoremstyle{remark}
\newtheorem*{exemple}{Example}
\newtheorem*{exemples}{Examples}
\newtheorem*{remarque}{Remark}
\title{The Central Limit Theorem}
\author{Olivier Castéra}
\email{o.castera@free.fr}
\urladdr{https://sciences-physiques.neocities.org}
\date{\today}
\begin{document}
\begin {abstract}
The central limit theorem states that the normal distribution appears whenever the observed random quantity can be presented as the sum of a sufficiently large number of independent or weakly linked elementary components, each of which has little influence on the sum.
\end {abstract}
\maketitle
\thispagestyle{empty}% remove page number
\tableofcontents
\section{Random Variable. Expectation. Moments. Variance.}
\begin{defbox}\begin{definition}
We call a discrete random variable $X$ a function that associates to each outcome of an experiment an unknown value $x_{i}$ in advance, from a set of possible values $x_{1},\dots, x_{n}$.
\end{definition}\end{defbox}
\begin{defbox}\begin{definition}
We call a continuous random variable $X$ a function that associates to each outcome of an experiment an unknown value $x$ in advance, over a finite or infinite set of possible values $[a,b]$.
\end{definition}\end{defbox}
\begin{defbox}\begin{definition}\label{defesp}
We call the mathematical expectation or mean value of the discrete random variable $X$ the sum of the products of all possible values $x_{i}$ of this variable by the probabilities $p_{i}$ of these values.
It is denoted $E[X]$ or $M[X]$ or $m_{x}$ or also $\bar x$. We have
\begin{equation*}
E[X]\overset{\Delta}{=}\sum_{i=1}^{n}x_{i}p_{i}
\end{equation*}
In the case of a continuous random variable $X$, we have
\begin{equation*}
E[X]\overset{\Delta}{=}\int_{-\infty}^{+\infty}xf(x)dx
\end{equation*}
\end{definition}\end{defbox}
\begin{defbox}\begin{definition}
We call the initial moment of order $s$ of a discrete random variable $X$ the function $\alpha_{s}[X]$ defined by
\begin{equation*}
\alpha_{s}[X]\overset{\Delta}{=}\sum_{i=1}^{n}x_{i}^{s}\,\mathcal{P}{i}
\end{equation*}
Initial meaning that the moment is calculated with respect to the origin of coordinates.
In the case of a continuous random variable $X$, we have
\begin{equation*}
\alpha{s}[X]\overset{\Delta}{=}\int_{-\infty}^{+\infty}x^{s}f(x)dx
\end{equation*}
\end{definition}\end{defbox}
Therefore, the first-order moment, $\alpha_{1}[X]$, coincides with the mathematical expectation $E[X]$ of the random variable $X$
\begin{equation*}
\alpha_{1}[X]\overset{\Delta}{=}E[X]
\end{equation*}
The moments characterize the probability law $\mathcal{P}$,
by giving its position, its degree of dispersion, and its shape.
\begin{defbox}\begin{definition}
We call the centered random variable associated with $X$, denoted $\mathring{X}$, the difference
\begin{equation*}
\mathring{X}\overset{\Delta}{=}X-E[X]
\end{equation*}
\end{definition}\end{defbox}
\begin{defbox}\begin{definition}
We call the centered moment of order $s$ of a discrete random variable $X$ the function $\mu_{s}[X]$ defined by
\begin{equation*}
\mu_{s}[X]\overset{\Delta}{=}\sum_{i=1}^{n}(x_{i}-m_{x})^{s}\,\mathcal{P}{i}
\end{equation*}
In the case of a continuous random variable $X$, we have
\begin{equation*}
\mu{s}[X]\overset{\Delta}{=}\int_{-\infty}^{+\infty}(x_{i}-m_{x})^{s}f(x)dx
\end{equation*}
\end{definition}\end{defbox}
\begin{thbox}\begin{theoreme}\label{thmc}
For any random variable, the first-order centered moment is zero.
\end{theoreme}\end{thbox}
\begin{proof}
\begin{align*}
E[\mathring X]&=E[X-E[X]]\\
&=E[X]-E[X]\\
&=0\qedhere
\end{align*}
\end{proof}
\begin{defbox}\begin{definition}\label{defvar}
We call the variance of the discrete random variable~$X$ its second-order centered moment. It is denoted $V[X]$ or $V_{x}$ or $D[X]$ or~$D_{x}$.
\begin{equation*}
V[X]\overset{\Delta}{=}\sum_{i=0}^{n}\mathring{x_{i}}^{2}\,\mathcal{P}{i}
\end{equation*}
In the case of a continuous random variable~$X$, we have
\begin{equation*}
V[X]\overset{\Delta}{=}\int_{-\infty}^{+\infty}\mathring{x_{i}}^{2}f(x)dx
\end{equation*}
\end{definition}\end{defbox}
By its definition, the variance is the mathematical expectation of the square
of the centered random variable associated with $X$.
\begin{equation*}
V[X]\overset{\Delta}{=}E[\mathring X^{2}]
\end{equation*}
\section{Bienaymé-Chebyshev Inequality}
\begin{thbox}\begin{theoreme}
Let $X$ be a random variable with mathematical expectation $E[X]$ and variance $V[X]$. For any positive real $\alpha$,
the probability that $X$ deviates from its mathematical expectation by an amount greater than or equal to $\alpha$
has an upper bound of $V[X]/\alpha^{2}$:
\begin{equation*}
P(|X-E[X]|\geqslant\alpha)\leqslant\dfrac{V[X]}{\alpha^{2}}
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
First, let's prove it for a discrete random variable $X$,
whose distribution is given by:
\begin{equation*}
\dfrac{x_{1}\ x_{2}\ x_{3}\ \dots\ x_{n}}{p_{1}\ p_{2}\ p_{3}\ \dots\ p_{n}}
\end{equation*}
where $p_{i}=P(X=x_{i})$ is the probability that the random variable $X$ takes the value $x_{i}$.
Let $\alpha$ be a positive real,
\begin{equation*}
P(|X-E[X]|\geqslant\alpha)=\sum_{|x_{i}-E[X]|\geqslant\alpha}p_{i}
\end{equation*}
Now,
\begin{align*}
V[X]&=\sum^{n}{i=1}(x_{i}-E[X])^{2}p_{i}\\
&=\sum^{n}{i=1}|x_{i}-E[X]|^{2}p_{i}
\end{align*}
The terms of the sum being positive or zero, we have
\begin{equation*}
V[X]\geqslant\sum_{|x_{i}-E[X]|\geqslant\alpha}|x_{i}-E[X]|^{2}p_{i}
\end{equation*}
and since $|x_{i}-E[X]|\geqslant\alpha$, we have
\begin{align*}
V[X]&\geqslant\sum_{|x_{i}-E[X]|\geqslant\alpha}\alpha^{2}p_{i}\\
\sum_{|x_{i}-E[X]|\geqslant\alpha}p_{i}&\leqslant\dfrac{V[X]}{\alpha^{2}}\\
P(|x-E[X]|\geqslant\alpha)&\leqslant\dfrac{V[X]}{\alpha^{2}}\qedhere
\end{align*}
\end{proof}

\begin{proof}
In the case where $X$ is continuous, let $f(x)$ be the probability density of $X$:
\begin{equation*}
P(|X-E[X]|\geqslant\alpha)=\int_{|x_{i}-E[X]|\geqslant\alpha}f(x)dx
\end{equation*}
Now,
\begin{align*}
V[X]&=\int^{+\infty}_{-\infty}(x_{i}-E[X])^{2}f(x)dx\\
&=\int^{+\infty}_{-\infty}|x_{i}-E[X]|^{2}f(x)dx
\end{align*}
The terms of the integral being positive or zero, we have
\begin{equation*}
V[X]\geqslant\int_{|x_{i}-E[X]|\geqslant\alpha}|x_{i}-E[X]|^{2}f(x)dx
\end{equation*}
and since $|x_{i}-E[X]|\geqslant\alpha$, we have
\begin{align*}
V[X]&\geqslant\int_{|x_{i}-E[X]|\geqslant\alpha}\alpha^{2}f(x)dx\\
\int_{|x_{i}-E[X]|\geqslant\alpha}f(x)dx&\leqslant\dfrac{V[X]}{\alpha^{2}}\\
P(|X-E[X]|\geqslant\alpha)&\leqslant\dfrac{V[X]}{\alpha^{2}}\qedhere
\end{align*}
\end{proof}

\section{Functions of Random Variables}
Let $X$ be a discrete random variable with the following distribution table
\begin{equation*}
\dfrac{x_{1}\ x_{2}\ x_{3}\ \dots\ x_{n}}{p_{1}\ p_{2}\ p_{3}\ \dots\ p_{n}}
\end{equation*}
Let $Z$ be a function of the random variable $X$
\begin{equation*}
Z=\varphi(X)
\end{equation*}
its distribution table is then the following
\begin{align*}
\dfrac{\varphi(x_{1})\ \varphi(x_{2})\ \varphi(x_{3})\ \dots\
\varphi(x_{n})}{\ p_{1}\ \ \ \ \ p_{2}\ \ \ \ \ p_{3}\ \ \ \dots\ \ \ \ p_{n}}
\end{align*}
Its mathematical expectation~$E[Z]$ is given by
\begin{align*}
E[Z]&=E[\varphi(X)]\\
&=\sum_{i=1}^{n}\varphi(x_{i})p_{i}
\end{align*}
When the random variable $X$ is continuous, we have
\begin{equation*}
E[Z]=\int_{-\infty}^{+\infty}\varphi(x)f(x)dx
\end{equation*}
When $Z$ is a function of two random variables, $X$ and $Y$
\begin{equation*}
Z=\varphi(X,Y)
\end{equation*}
its distribution table is the following
\begin{align*}
&\dfrac{\varphi(x_{1},y_{1})\ \varphi(x_{1},y_{2})\ \varphi(x_{1},y_{3})\ \dots\ \varphi(x_{1},y_{n})}
{p_{11}\ \ \ \ \ \ \ \ p_{12}\ \ \ \ \ \ \ \ p_{13}\ \ \ \ \ \dots\ \ \ \ p_{1n}}\\
&\dfrac{\varphi(x_{2},y_{1})\ \varphi(x_{2},y_{2})\ \varphi(x_{2},y_{3})\ \dots\ \varphi(x_{2},y_{n})}
{p_{21}\ \ \ \ \ \ \ \ p_{22}\ \ \ \ \ \ \ \ p_{23}\ \ \ \ \ \dots\ \ \ \ p_{2n}}\\
&\dfrac{\varphi(x_{3},y_{1})\ \varphi(x_{3},y_{2})\ \varphi(x_{3},y_{3})\ \dots\ \varphi(x_{3},y_{n})}
{p_{31}\ \ \ \ \ \ \ \ p_{32}\ \ \ \ \ \ \ \ p_{33}\ \ \ \ \ \dots\ \ \ \ p_{3n}}\\&\dots\\
&\dfrac{\varphi(x_{n},y_{1})\ \varphi(x_{n},y_{2})\ \varphi(x_{n},y_{3})\ \dots\ \varphi(x_{n},y_{n})}
{p_{n1}\ \ \ \ \ \ \ \ p_{n2}\ \ \ \ \ \ \ \ p_{n3}\ \ \ \ \ \dots\ \ \ \ p_{nn}}\\
\end{align*}
where $p_{ij}=P[(X=x_{i})(Y=y_{j})]$ is the probability that the system $(X,Y)$ takes the value $(x_{i},y_{j})$.
Its mathematical expectation~$E[Z]$ is given by
\begin{align}
E[Z]&=E[\varphi(X,Y)]\notag\\
&=\sum_{i=1}^{n}\sum_{j=1}^{n}\varphi(x_{i},y_{j})p_{ij}\label{Edisc}
\end{align}
When the random variables $X$ and $Y$ are continuous, we have
\begin{equation}
E[Z]=\iint_{-\infty}^{+\infty}\varphi(x,y)f(x,y)dxdy\label{Econt}
\end{equation}

\subsection{Sum of Two Random Variables}
\begin{thbox}\begin{theoreme}\label{thmoy2}
The mathematical expectation of the sum of two random variables is equal to the sum of their
mathematical expectations.
\begin{equation*}
E[X+Y]=E[X]+E[Y]
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
Let $X$ and $Y$ be two discrete random variables. With equation~\eqref{Edisc} we have
\begin{align*}
E[X+Y]&=\sum_{i=1}^{n}\sum_{j=1}^{n}(x_{i}+y_{j})p_{ij}\\
&=\sum_{i=1}^{n}\sum_{j=1}^{n}x_{i}p_{ij}+\sum_{i=1}^{n}\sum_{j=1}^{n}y_{j}p_{ij}\\
&=\sum_{i=1}^{n}\left( x_{i}\sum_{j=1}^{n}p_{ij}\right)+\sum_{j=1}^{n}\left(y_{j}\sum_{i=1}^{n}p_{ij}\right)\\
&=\sum_{i=1}^{n}x_{i}p_{i}+\sum_{j=1}^{n}y_{j}p_{j}\\
&=E[X]+E[Y]\qedhere
\end{align*}
\end{proof}
\begin{proof}
Now let $X$ and $Y$ be two continuous variables. With equation~\eqref{Econt} we have
\begin{align*}
E[X+Y]&=\iint_{-\infty}^{+\infty}(x+y)f(x,y)dxdy\\
&=\iint_{-\infty}^{+\infty}xf(x,y)dxdy+\iint_{-\infty}^{+\infty}yf(x,y)dxdy\\
&=\int_{-\infty}^{+\infty}x\left( \int_{-\infty}^{+\infty}f(x,y)dy\right)dx+
\int_{-\infty}^{+\infty}y\left( \int_{-\infty}^{+\infty}f(x,y)dx\right)dy\\
&=\int_{-\infty}^{+\infty}xf_{1}(x)dx+\int_{-\infty}^{+\infty}yf_{2}(y)dy\\
&=E[X]+E[Y]\qedhere
\end{align*}
\end{proof}
\begin{defbox}\begin{definition}\label{defcov}
We call the covariance of the random variables $X$ and $Y$ the mathematical expectation of the product
of the two centered random variables $\mathring X$ and $\mathring Y$ associated with $X$ and $Y$. It is denoted~$K_{xy}$
\begin{equation*}
K_{xy}\overset{\Delta}{=}E[\mathring X\mathring Y]
\end{equation*}
\end{definition}\end{defbox}
\begin{thbox}\begin{theoreme}\label{thvar2}
The variance of the sum of two random variables is equal to the sum of their
variances plus twice the covariance.
\begin{equation*}
V[X+Y]=V[X]+V[Y]+2K_{xy}
\end{equation*}
\begin{proof}
Let the random variable $Z$ be such that
\begin{equation*}
Z=X+Y
\end{equation*}
Using theorem \ref{thmoy2}, we have
\begin{align*}
E[Z]&=E[X+Y]\\
&=E[X]+E[Y]
\end{align*}
Thus
\begin{align*}
Z-E[Z]&=X-E[X]+Y-E[Y]\\
\mathring Z&=\mathring X+\mathring Y
\end{align*}
By definition of variance
\begin{align*}
V[Z]&=E[\mathring Z^{2}]\\
&=E[(\mathring X+\mathring Y)^{2}]\\
&=E[\mathring X^{2}+\mathring Y^{2}+2\mathring X\mathring Y]\\
&=E[\mathring X^{2}]+E[\mathring Y^{2}]+2E[\mathring X\mathring Y]\\
&=V[X]+V[Y]+2K_{xy}\qedhere
\end{align*}
\end{proof}
\end{theoreme}\end{thbox}
\begin{defbox}\begin{definition}\label{defind}
The random variables $X$ and $Y$ are said to be independent if the distribution law of each does not
depend on the values taken by the other.\\
The probability of a system of discrete independent random variables is
equal to the product of the probabilities of the variables in the system
\begin{align*}
P_{ij}&=P[(X=x_{i})(Y=y_{j})]\\
&=P[(X=x_{i})]P[(Y=y_{j})]\\
&=P_{i}P_{j}
\end{align*}
The probability density of a system of continuous independent random variables is
equal to the product of the densities of the variables in the system
\begin{equation*}
f(x,y)=f_{1}(x)f_{2}(y)
\end{equation*}
\end{definition}\end{defbox}
\begin{thbox}\begin{theoreme}\label{thcovind}
The covariance $K_{xy}$ of independent random variables $X$ and $Y$ is zero.
\end{theoreme}\end{thbox}
\begin{proof}
Consider discrete random variables
\begin{equation*}
K_{xy}=\sum_{i=1}^{n}\sum_{j=1}^{n}(x_{i}-m_{x})(y_{j}-m_{y})P_{ij}
\end{equation*}
From definition \eqref{defind} of independent variables
\begin{equation*}
P_{ij}=P_{i}P_{j}
\end{equation*}
Thus,
\begin{equation*}
K_{xy}=\sum_{i=1}^{n}(x_{i}-m_{x})P_{i}\sum_{j=1}^{n}(y_{j}-m_{y})P_{j}
\end{equation*}
These sums are respectively the first-order centered moment of $X$ and $Y$, which are zero
from theorem \ref{thmc}, and
\begin{equation*}
K_{xy}=0\qedhere
\end{equation*}
\end{proof}
\begin{proof}
Now consider continuous random variables
\begin{equation*}
K_{xy}=\iint_{-\infty}^{+\infty}(x-m_{x})(y-m_{y})f(x,y)dxdy
\end{equation*}
From definition \eqref{defind} of independent variables
\begin{equation*}
f(x,y)=f_{1}(x)f_{2}(y)
\end{equation*}
Thus,
\begin{equation*}
K_{xy}=\int_{-\infty}^{+\infty}(x-m_{x})f_{1}(x)dx\int_{-\infty}^{+\infty}(y-m_{y})f_{2}(y)dy
\end{equation*}
These integrals are respectively the first-order centered moment of $X$ and $Y$, which are zero
from theorem \ref{thmc}, and
\begin{equation*}
K_{xy}=0\qedhere
\end{equation*}
\end{proof}
\subsection{Product of Two Random Variables}
\begin{thbox}\begin{theoreme}\label{thespprod}
The mathematical expectation of the product of two random variables $X$ and $Y$ is equal to the product of their expectations
plus the covariance
\begin{equation*}
E[XY]=E[X]E[Y]+K_{xy}
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
From definition \eqref{defcov} of covariance
\begin{align*}
K_{xy}&=E[\mathring X\mathring Y]\\
&=E[(X-E[X])(Y-E[Y])]\\
&=E[XY-XE[Y]-E[X]Y+E[X]E[Y]]\\
&=E[XY]-E[XE[Y]]-E[E[X]Y]+E[E[X]E[Y]]\\
&=E[XY]-E[X]E[Y]-E[X]E[Y]+E[X]E[Y]\\
&=E[XY]-E[X]E[Y]\qedhere
\end{align*}
\end{proof}
\section{Central Limit Theorem}
\subsection{Characteristic Function}
\begin{defbox}\begin{definition}\label{deffc}
We call the characteristic function of the discrete random variable $X$ the mathematical expectation of the function $\exp{(itX)}$
\begin{align*}
g(t)&=\sum_{k=1}^{n}e^{itx_{k}}p_{k}\\
&=E\left[ e^{itX}\right]
\end{align*}
We call the characteristic function of the continuous random variable $X$ the Fourier transform
of its probability density~$f(x)$
\begin{align*}
g(t)&=\int_{-\infty}^{+\infty}e^{itx}f(x)dx\\
&=E\left[ e^{itX}\right]
\end{align*}
\end{definition}\end{defbox}
\begin{thbox}\begin{theoreme}\label{tha}
If two random variables $X$ and $Y$ are related by the relation
\begin{equation*}
Y=aX
\end{equation*}
where $a$ is a non-random factor, their characteristic functions are related by the relation
\begin{equation*}
g_{y}(t)=g_{x}(at)
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
\begin{align*}
g_{y}(t)&=E\left[ e^{itY}\right] \\
&=E\left[ e^{itaX}\right] \\
&=E\left[ e^{i(at)X}\right] \\
&=g_{x}(at)\qedhere
\end{align*}
\end{proof}
\begin{thbox}\begin{theoreme}\label{thcara}
The characteristic function of a sum of independent random variables is equal to the product of the characteristic
functions of the components.
\end{theoreme}\end{thbox}
\begin{proof}
Let $X_{1},X_{2},\dots,X_{n}$ be independent random variables with $g_{x_{1}},g_{x_{2}},\dots,g_{x_{n}}$ for characteristic
functions, and sum
\begin{equation*}
Y=\sum_{k=1}^{n}X_{k}
\end{equation*}
The characteristic function of $Y$ is written
\begin{align*}
g_{y}(t)&=E\left[ e^{itY}\right] \\
&=E\left[ e^{it\sum_{k=1}^{n}X_{k}}\right] \\
&=E\left[\prod_{k=1}^{n}e^{itX_{k}}\right]
\end{align*}
The random variables $X_{k}$ being independent, the functions $e^{itX_{k}}$ are also independent and we can
use theorem \ref{thespprod}
\begin{align*}
g_{y}(t)&=\prod_{k=1}^{n}E\left[ e^{itX_{k}}\right] \\
&=\prod_{k=1}^{n}g_{x_{k}}(t)\qedhere
\end{align*}
\end{proof}
\subsection{Characteristic Function of the Normal Distribution}
\begin{thbox}\begin{theoreme}\label{thfcln}
The characteristic function of the normal distribution with mathematical expectation $m_{x}$
zero and variance $\sigma^{2}$ is written
\begin{equation*}
g(t)=e^{-t^{2}\sigma^{2}/2}
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
Let a continuous random variable $X$ be distributed according to the normal distribution,
with mathematical expectation $m_{x}$ zero, and variance $\sigma^{2}$.
Its probability density $f(x)$ is written
\begin{equation*}
f(x)=\dfrac{1}{\sigma\sqrt{2\pi}}e^{-x^{2}/(2\sigma^{2})}
\end{equation*}
Its characteristic function is written
\begin{align*}
g(t)&=\dfrac{1}{\sigma\sqrt{2\pi}}\int_{-\infty}^{+\infty}e^{itx}e^{-x^{2}/(2\sigma^{2})}dx\\
&=\dfrac{1}{\sigma\sqrt{2\pi}}\int_{-\infty}^{+\infty}e^{itx-x^{2}/(2\sigma^{2})}dx
\end{align*}
We use the following formula
\begin{equation*}
\int_{-\infty}^{+\infty}e^{-Ax^{2}\pm2Bx-C},dx=\sqrt{\dfrac{\pi}{A}},e^{-(AC-B^{2})/A}
\end{equation*}
We identify: $A=1/(2\sigma^{2})$, $B=it/2$, and $C=0$. Hence
\begin{align*}
g(t)&=\dfrac{1}{\sigma\sqrt{2\pi}}\sqrt{2\pi\sigma^{2}}\,e^{-(t^{2}/4)2\sigma^{2}}\\
&=e^{-t^{2}\sigma^{2}/2}\qedhere
\end{align*}
\end{proof}
\subsection{Sum of Two Normal Distributions}
\begin{thbox}\begin{theoreme}\label{thsdln}
The sum of two normal distributions is a normal distribution.
\end{theoreme}\end{thbox}
\begin{proof}
Let $X$ and $Y$ be two independent continuous random variables whose probability
densities are distributed according to normal distributions
\begin{align*}
f(x)=\dfrac{1}{\sigma_{x}\sqrt{2\pi}}\,e^{-x^{2}/(2\sigma_{x}^{2})}\\
f(y)=\dfrac{1}{\sigma_{y}\sqrt{2\pi}}\,e^{-y^{2}/(2\sigma_{y}^{2})}
\end{align*}
From definition \ref{deffc}, their characteristic functions are written
\begin{align*}
g_{x}(t)&=e^{-t^{2}\sigma_{x}^{2}/2}\\
g_{y}(t)&=e^{-t^{2}\sigma_{y}^{2}/2}
\end{align*}
From theorem \ref{thcara}, the characteristic function $g_{z}(t)$ of their sum is equal
to the product of the characteristic functions
\begin{align*}
g_{z}(t)&=g_{x}(t)\times g_{y}(t)\\
&=e^{-t^{2}\sigma_{x}^{2}/2}\times e^{-t^{2}\sigma_{y}^{2}/2}\\
&=e^{-t^{2}(\sigma_{x}^{2}+\sigma_{y}^{2})/2}
\end{align*}
which, from theorem \ref{thfcln}, is the characteristic function of a normal distribution with mathematical expectation~$m_{z}$ zero and
variance $\sigma_{x}^{2}+\sigma_{y}^{2}$.
\end{proof}
\subsection{Central Limit Theorem}
\begin{thbox}\begin{theoreme}
Let $X_{1},X_{2},\dots,X_{n}$ be independent random variables with the same distribution law,
with mathematical expectation~$m_{x}$ and variance~$\sigma^{2}$. As $n$ increases indefinitely,
the distribution law of the sum
\begin{equation*}
Y_{n}=\dfrac{1}{\sqrt{n}}\,\sum_{k=1}^{n}X_{k}
\end{equation*}
tends toward the normal distribution.
\end{theoreme}\end{thbox}
\begin{proof}
Note that the factor $1/\sqrt{n}$ is necessary for the proof, but the convergence of $Y_{n}$ toward the
normal distribution implies that of the sum of the random variables $\sum_{k=1}^{n}X_{k}$. We will use the
characteristic functions and admit without proof that
the convergence of the characteristic functions implies that of the laws.
From theorem \ref{tha}, the characteristic function of $Y_{n}$ is equal to
\begin{align*}
g_{y_{n}}(t)&=g_{1/\sqrt{n}\,\sum_{k=1}^{n}X_{k}}\\
&=g_{\sum_{k=1}^{n}X_{k}}\left( \dfrac{t}{\sqrt{n}}\right)
\end{align*}
From theorem \ref{thcara}, the characteristic function of $Y_{n}$ is equal to the product of the
characteristic functions of the $X_{k}$
\begin{equation}
g_{y_{n}}(t)=\left[ g_{x}\left( \dfrac{t}{\sqrt{n}}\right) \right] ^{n}\label{gy}
\end{equation}
The characteristic function of each $X_{k}$ is written
\begin{equation*}
g_{x}\left( \dfrac{t}{\sqrt{n}}\right)=\int_{-\infty}^{+\infty}e^{ixt/\sqrt{n}}f(x)dx
\end{equation*}
Perform a Taylor expansion around zero, up to order $3$
\begin{align}
&g_{x}\left( \dfrac{t}{\sqrt{n}}\right)=g_{x}(0)+g'_{x}(0)\left( \dfrac{t}{\sqrt{n}}\right)+\left[ \dfrac{g''_{x}(0)}{2}+o\left( \dfrac{t}{\sqrt{n}}\right)\right]\dfrac{t^{2}}{n}\label{gx}\\
&\text{where}\ \lim_{t/\sqrt n\to 0} o\left( \dfrac{t}{\sqrt{n}}\right)=0\notag
\end{align}
Calculate each term
\begin{align*}
g_{x}(0)&=\int_{-\infty}^{+\infty}f(x)dx\\
&=1
\end{align*}
By differentiating we have
\begin{align*}
g'_{x}\left(\dfrac{t}{\sqrt{n}}\right)&=\int_{-\infty}^{+\infty}ixe^{ixt/\sqrt{n}}f(x)dx\\
&=i\int_{-\infty}^{+\infty}xe^{ixt/\sqrt{n}}f(x)dx
\end{align*}
setting $t/\sqrt n=0$ we have
\begin{align*}
g'_{x}(0)&=i\int_{-\infty}^{+\infty}xf(x)dx\\
&=im_{x}
\end{align*}
We do not restrict generality by shifting the origin to the point $m_{x}$, then we have $m_{x}=0$, and
\begin{equation*}
g'_{x}(0)=0
\end{equation*}
Differentiating a second time
\begin{equation*}
g''_{x}\left( \dfrac{t}{\sqrt{n}}\right)=-\int_{-\infty}^{+\infty}x^{2}e^{ixt/\sqrt{n}}f(x)dx
\end{equation*}
setting $t/\sqrt n=0$
\begin{equation*}
g''_{x}(0)=-\int_{-\infty}^{+\infty}x^{2}f(x)dx
\end{equation*}
and since we set $E[X]=0$, from definition \ref{defvar} the previous expression is nothing other
than the negative of the variance
\begin{equation*}
g''_{x}(0)=-\sigma^{2}
\end{equation*}
Substituting these results into equation \eqref{gx}, when $t/\sqrt n\to 0$
\begin{equation*}
g_{x}\left( \dfrac{t}{\sqrt{n}}\right)=1-\left[ \dfrac{\sigma^{2}}{2}-o\left( \dfrac{t}{\sqrt{n}}\right)\right]\dfrac{t^{2}}{n}
\end{equation*}
then into equation \eqref{gy}
\begin{equation*}
g_{y_{n}}(t)=\left\lbrace 1-\left[ \dfrac{\sigma^{2}}{2}-o\left( \dfrac{t}{\sqrt{n}}\right)\right]\dfrac{t^{2}}{n}\right\rbrace ^{n}
\end{equation*}
Take the logarithm of this expression
\begin{equation*}
\ln g_{y_{n}}(t)=n\ln\left\lbrace 1-\left[ \dfrac{\sigma^{2}}{2}-o\left( \dfrac{t}{\sqrt{n}}\right)\right]\dfrac{t^{2}}{n}\right\rbrace
\end{equation*}
Set
\begin{equation*}
\chi=\left[ \dfrac{\sigma^{2}}{2}-o\left( \dfrac{t}{\sqrt{n}}\right) \right]\dfrac{t^{2}}{n}
\end{equation*}
and we have
\begin{equation*}
\ln g_{y_{n}}(t)=n\ln\left\lbrace 1-\chi\right\rbrace
\end{equation*}
As $n$ tends to infinity, $\chi$ tends to $0$, and $\ln(1-\chi)$ tends to $-\chi$
\begin{align*}
\lim_{n\to +\infty}\ln g_{y_{n}}(t)&=\lim_{n\to +\infty}n(-\chi)\\
&=\lim_{n\to +\infty} n\left[ -\dfrac{\sigma^{2}}{2}+
o\left( \dfrac{t}{\sqrt{n}}\right) \right]\dfrac{t^{2}}{n}\\
&=-\dfrac{t^{2}\sigma^{2}}{2}+\lim_{n\to+\infty}t^{2},o\left( \dfrac{t}{\sqrt{n}}\right)
\end{align*}
Now $\lim_{t/\sqrt n\to0}o(t)=0$, so
\begin{equation*}
\lim_{n\to+\infty}o\left( \dfrac{t}{\sqrt{n}}\right)=0
\end{equation*}
thus
\begin{equation*}
\lim_{n\to +\infty}\ln g_{y_{n}}(t)=-\dfrac{t^{2}\sigma^{2}}{2}
\end{equation*}
hence
\begin{equation*}
\lim_{n\to +\infty}g_{y_{n}}(t)=e^{-t^{2}\sigma^{2}/2}
\end{equation*}
which, from theorem \ref{thfcln}, is the characteristic function of the normal distribution with mathematical expectation zero,
and variance $\sigma^{2}$.
\end{proof}
\section{Laws of Large Numbers}
The laws of large numbers assert the convergence in probability of random variables toward constant quantities,
as the number of experiments increases indefinitely.
\subsection{Chebyshev's Theorem}
Let a random variable $X$ provide, during $n$ independent experiments, the values
$x_{1},x_{2},\dots,x_{n}$. To use theorem \ref{thmoy2}, we will consider that these values
are the results of a single experiment performed on each of the $n$ independent random variables
$X_{1},X_{2},\dots,X_{n}$,
with the same distribution law as the random variable $X$.\\
Rather than considering the sum of these random variables,
we will consider the arithmetic mean of these variables, in other words the sum divided by $n$.
Let the random variable $Y$ be defined as the arithmetic mean of these $n$ random variables
\begin{equation*}
Y=\dfrac{1}{n}\sum_{i=1}^{n}X_{i}
\end{equation*}
Its mathematical expectation is written
\begin{align*}
E[Y]&=E\left[\dfrac{1}{n}\sum_{i=1}^{n}X_{i}\right]\\
&=\dfrac{1}{n}\,E\left[\sum_{i=1}^{n}X_{i}\right]
\end{align*}
With theorem \ref{thmoy2} we have
\begin{align*}
E[Y]&=\dfrac{1}{n}\sum_{i=1}^{n}E[X_{i}]\\
&=\dfrac{1}{n}\,n\,E[X]\\
&=E[X]
\end{align*}
The variance of $Y$ is written
\begin{align*}
V[Y]&=E[\mathring Y^{2}]\\
&=E\left[\left( \dfrac{1}{n}\sum_{i=1}^{n}\mathring X_{i}\right)^{2}\right]\\
&=\dfrac{1}{n^{2}}\,E\left[\left(\sum_{i=1}^{n}\mathring X_{i}\right)^{2}\right]\\
&=\dfrac{1}{n^{2}}\,V\left[\sum_{i=1}^{n}X_{i}\right]
\end{align*}
With theorem \ref{thvar2} and theorem \ref{thcovind} we have
\begin{align*}
V[Y]&=\dfrac{1}{n^{2}}\sum_{i=1}^{n}V[X_{i}]\\
&=\dfrac{1}{n}\,V[X]
\end{align*}
\begin{thbox}\begin{theoreme}\label{thtch}
For a sufficiently large number of experiments, the arithmetic mean of the observed values
of a random variable converges in probability toward its mathematical expectation
\begin{equation*}
\forall \delta>0,\,\forall \varepsilon >0,\ \exists n\ \text{such that}\
P\left( \left |\dfrac{1}{n}\sum_{i=1}^{n}X_{i}-E[X]\right|\geqslant\varepsilon\right)<\delta
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
Apply to $Y$ the Bienaymé-Chebyshev inequality by setting $\alpha=\varepsilon$
\begin{equation*}
P(|Y-E[Y]|\geqslant\varepsilon)\leqslant\dfrac{V[Y]}{\varepsilon^{2}}
\end{equation*}
Applying the previous results on the expectation and variance of $Y$
\begin{equation*}
P\left(\left|\dfrac{1}{n}\sum_{i=1}^{n}X_{i}-E[X]\right|\geqslant\varepsilon\right)\leqslant\dfrac{V[X]}{n\varepsilon^{2}}
\end{equation*}
and set $\delta=V[X]/(n\varepsilon^{2})$. We can always choose $n$ sufficiently large
to have the above inequality, no matter how small $\varepsilon$ is.
\end{proof}
\subsection{Generalized Chebyshev Theorem}
We can generalize the law of large numbers to the case of random variables with different distribution laws.
\begin{thbox}\begin{theoreme}
Let there be $n$ independent random variables $X_{1},$ $X_{2},$ $\dots,$ $X_{n}$, with different distribution laws,
with mathematical expectations $E[X_{1}],$ $E[X_{2}],$ $\dots,$ $E[X_{n}]$, and variances $V[X_{1}],V[X_{2}],\dots,V[X_{n}]$.
If all variances have the same upper bound $L$
\begin{equation*}
\forall i,\ V[X_{i}]<L
\end{equation*}
for a sufficiently large number of experiments, the arithmetic mean of the observed values of the random
variables $X_{1},X_{2},\dots,X_{n}$, converges in probability toward the arithmetic mean of their mathematical expectations
\begin{equation*}
\forall \delta>0,\,\forall \varepsilon >0,\ \exists n\ \text{such that}\quad
P\left( \left|\dfrac{1}{n}\sum_{i=1}^{n}X_{i}-\dfrac{1}{n}\sum_{i=1}^{n}E[X_{i}] \right|\geqslant\varepsilon \right)<\delta
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
Let the random variable $Y$ be such that
\begin{equation*}
Y=\dfrac{1}{n}\sum_{i=1}^{n}X_{i}
\end{equation*}
Apply to $Y$ the Chebyshev inequality:
\begin{align*}
\forall \delta>0,\,\forall \varepsilon >0,\ \exists n\ \text{such that}\quad P\left(|Y-E[Y]|\geqslant\varepsilon\right)&<\dfrac{V[Y]}{\varepsilon^{2}}\\
P\left(\left|\dfrac{1}{n}\sum_{i=1}^{n}X_{i}-\dfrac{1}{n}\sum_{i=1}^{n}E[X_{i}]\right|\geqslant\varepsilon\right)
&<\dfrac{\sum_{i=1}^{n}V[X_{i}]}{n^{2}\varepsilon^{2}}\\
P\left(\left|\dfrac{1}{n}\sum_{i=1}^{n}X_{i}-\dfrac{1}{n}\sum_{i=1}^{n}E[X_{i}]\right|\geqslant\varepsilon\right)
&<\dfrac{L}{n\varepsilon^{2}}
\end{align*}
and set $\delta=L/(n\varepsilon^{2})$. We can always choose $n$ sufficiently large
to have the above inequality, no matter how small $\varepsilon$ is.
\end{proof}
\subsection{Markov's Theorem}
We can generalize the law of large numbers to the case of dependent random variables.
\begin{thbox}\begin{theoreme}Markov's Theorem\
Let there be $n$ dependent random variables $X_{1},$ $X_{2},$ $\dots,$ $X_{n}$, with different distribution laws,
with mathematical expectations $E[X_{1}],E[X_{2}],\dots,E[X_{n}]$, and variances $V[X_{1}],V[X_{2}],\dots,V[X_{n}]$.
If
\begin{equation*}
\lim_{n\to \infty}\dfrac{1}{n^{2}}\sum_{i=1}^{n}V[X_{i}]=0
\end{equation*}
the arithmetic mean of the observed values of the random variables
$X_{1},X_{2},\dots,X_{n}$, converges in probability toward the arithmetic mean of their mathematical expectations
\begin{equation*}
\forall \delta>0,\,\forall \varepsilon >0,\ \exists n\ \text{such that}\quad
P\left( \left|\dfrac{1}{n}\sum_{i=1}^{n}X_{i}-\dfrac{1}{n}\sum_{i=1}^{n}E[X_{i}] \right|\geqslant\varepsilon \right)<\delta
\end{equation*}
\end{theoreme}\end{thbox}
\begin{proof}
Let the random variable $Y$ be such that
\begin{equation*}
Y=\dfrac{1}{n}\sum_{i=1}^{n}X_{i}
\end{equation*}
Apply to $Y$ the Chebyshev inequality:
\begin{align*}
\forall \delta>0,\,\forall \varepsilon >0,\ \exists n\ \text{such that}\quad P\left(|Y-E[Y]|\geqslant\varepsilon\right)&<\dfrac{V[Y]}{\varepsilon^{2}}\\
P\left(\left|\dfrac{1}{n}\sum_{i=1}^{n}X_{i}-\dfrac{1}{n}\sum_{i=1}^{n}E[X_{i}]\right|\geqslant\varepsilon\right)
&<\dfrac{\sum_{i=1}^{n}V[X_{i}]}{n^{2}\varepsilon^{2}}
\end{align*}
Set $\delta=\dfrac{1}{n^{2}\varepsilon^{2}}\sum_{i=1}^{n}V[X_{i}]$.
By hypothesis, we can always choose $n$ sufficiently large to have ${\lim_{n\to \infty}\delta=0}$, no matter how small $\varepsilon$ is.
\end{proof}

\end{document}
