\documentclass[11pt]{article}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc}

\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{microtype}
\usepackage{booktabs}
\usepackage{url}
\usepackage{algorithm}
\usepackage[noend]{algpseudocode}
\usepackage{wasysym}
\usepackage{amssymb }
\usepackage{caption}
\usepackage{tikz}

%\usepackage{fontspec}
%\setmainfont{Hoefler Text}

\newcommand*\Let[2]{\State #1 $\gets$ #2}
%\algrenewcommand\alglinenumber[1]{
   % {\sf\footnotesize\addfontfeatures{Colour=888888,Numbers=Monospaced}#1}}
\algrenewcommand\algorithmicrequire{\textbf{Precondition:}}
\algrenewcommand\algorithmicensure{\textbf{Postcondition:}}

\RequirePackage[l2tabu, orthodox]{nag}


\usepackage{amsmath,amssymb,amsthm}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\epsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 388R: Randomized Algorithms } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex

%%% START

\begin{document}

\lecture{1 --- Aug. 26, 2015}{Fall 2015}{Prof.\ Eric Price}{Ashish Bora, Jessica Hoffmann}

\section{Overview}
In this lecture, we get a first taste of randomized algorithms, from definition to conception. We'll see different types of such algorithms, as well as two examples, QuickSort and Karger's Min-Cut.


\section{What are randomized algorithms?}
Randomized algorithms use randomization, which means they usually do something different if you use it different times.

\begin{center}
\begin{tikzpicture}
	\draw [->,>=stealth] (0,0) node [left] {input} -- (1,0);
	\draw (1,-0.5) rectangle (3.5,0.5);
	\node at (2.25,0) {Algorithm};
	\draw [->,>=stealth] (2.25,-1) node [below] {random bits} -- (2.25,-0.5);	
	\draw [->,>=stealth] (3.5,0) -- (4.5,0) node [right] {output};	
\end{tikzpicture}
\end{center}

\begin{description}
\item[Deterministic:] For all inputs, output is good.
\item[Random:] For all inputs, with good probability, output is good.
\item[Average case:] Define a distribution over inputs, the algorithm works with good probability.
\end{description}
\textit{Issue:} You usually don't have the distribution over the inputs, and we might lose structure by imposing a distribution. For instance, almost all matrices are invertible, but neglecting all non-invertible matrices might be super stupid for a given problem. Or okay. It depends.\\\\
\textbf{Examples of famous random algorithms:} 
\begin{itemize}
\item Quicksort
\item Bloom filters
\item Cuckoo hash, and hash tables in general
\item Routing packets
\end{itemize}

\section{Why randomize?}
The goals are simplicity, speed and sometimes other objectives depending on the problem (e.g., for Nash Equilibrium, the problem inherently requires randomization).\\\\
\textbf{Themes:} 
\begin{itemize}
\item Avoiding adversarial inputs (``adversarial" may mean well-structured, i.e. natural). Randomization makes your data look random, even if it is not (think Quicksort).
\item Finger printing (have a quick identifier for a complex structure)
\item Load balancing (distribute workload across resources to optimize time consumption, or efficiency, or avoid overloads)
\item Sampling (estimate properties of the whole population based on a subset)
\item Symmetry breaking (e.g. what to do when two packets arrive at the same time?)
\item Probabilistic existence proofs (Erd\"os, Lovasz... if a random object satisfies a property with non-zero probability, we just proved that such an object exists)
\end{itemize}


\section{Quicksort}
\begin{algorithm}
  \caption{You all know Quicksort. It's for sorting, but quickly.
    \label{QS}}
  \begin{algorithmic}[1]
    %\Require{$x$ and $y$ are packed \DNA strings of equal length $n$}
    \Statex
    \Function{Qsort}{List}
    \If{$List==[]$}
         \State \Return{[]}
    \EndIf
    \State Choose $t \in [n]$
    \State \Return{Qsort($ [x_i \in List | x_i < x_t] $) + $[x_t]$ + Qsort($[x_i \in List | x_i \geq x_t]$)}
    \EndFunction
  \end{algorithmic}
\end{algorithm}
Time $\propto$ $\# (i,j)$ that are compared.\\
Let us number the elements such that $x_i \leq x_j$ whenever $i < j$. Let $Z_{i,j}$ be the event that i and j are compared i.e. $Z_{i,j} = 1$ if $i$ and $j$ are compared, 0 otherwise.\\

\textbf{Notation:} $a \apprle b \iff \exists c \in \mathbb{R} , a \leq c \cdot b$\\\\
Time $\displaystyle\apprle \sum_{i<j} Z_{i,j}$.
What is $\mathbb{P}[Z_{i,j} = 1]$ ?\\
$Z_{i,j}$ represents the event that i or j is chosen before $i+1, i+2, \cdots, j-1$.

\begin{align*} 
\mathbb{P}[Z_{i,j}] &= \frac{2}{|j-i| + 1} = \mathbb{E}[Z_{i,j}] \\
\mathbb{E}[Time] &\apprle   \mathbb{E}\left[\sum_{i<j} Z_{i,j}\right] \\ 
 &= \sum_{i<j} \mathbb{E}\left[ Z_{i,j}\right] \\
 &= \sum_{i}\sum_{i<j} \mathbb{E}\left[ \frac{2}{j-i+ 1} \right] \\
 &= 2 \cdot \sum_{i} \mathbb{E}\left[ \frac{1}{2} + \dots + \frac{1}{n -i+ 1}\right] \\
 &\leq 2 \cdot n \cdot (H_n - 1) \\
 &\leq 2 \cdot n \cdot \log{n}
\end{align*}
\textit{Comment:} All of that is possible because the expectation is linear. Yeah, you're all blas\'e, you know that already. But look, $\sum Z_{i,j}$ is a super complex variable, many $ Z_{i,j}$'s are highly dependent on one another, and by using the expectation, suddenly we can treat them all separately. Don't be unimpressed, this is awesome!

\section{Karger's minimum cut ('93)}
Given a graph $G=(V,E)$ with n vertices and m edges, find a global minimum cut, therefore a set of vertices $S \subset V$, with $1 \leq |S| \leq n-1$, which minimizes the number of edges going from this subset S to the rest of the vertices. We denote the cut value for set $S$ by $|E(S, \bar{S})|$ (where $\bar{S} = V \setminus S$).

The problem of finding a minimum cut with a given source and sink ($s-t$ min cut) is probably more familiar. Dinic's algorithm or Ford-Fulkerson algorithm lead to a solution. We can compute a $s-t$ minimum cut in $\mathcal{O}(n^3)$. By running this algorithm for all $(s,t)$ pairs, we get a solution for the global minimum cut in $\mathcal{O}(n^5)$.

\begin{algorithm}[H]
  \caption{It looks stupid, but actually it's not \textit{that} stupid.
    \label{algo1}}
  \begin{algorithmic}[1]
    %\Require{$x$ and $y$ are packed \DNA strings of equal length $n$}
    \Statex
    \Function{Algo1}{}
    \While{$n>2$}
         \State Choose an edge $e=(u,v)$ randomly from the remaining edges
         \State Contract that edge
    \EndWhile
    \State \Return{the two last vertices}
    \EndFunction
  \end{algorithmic}
\end{algorithm}
Contracting an edge means that we remove that edge and combine the two vertices into a super-node. We note that self-loops thus formed are removed but any resulting parallel edges are \emph{not} removed. This means at every step, we have a multi-graph without any self-loops.

This algorithm will yield a cut. The minimum cut? Probably not. But maybe! If the graph is somewhat dense, then choosing an edge in the minimum cut is rather unlikely at the beginning...and very likely at the end.\\\\
Here is a successful run of this algorithm, courtesy to Wikipedia.
\begin{figure}[H]
    \centering
    \includegraphics[width=\textwidth]{algo1.png}
    \caption{A (very) lucky run [1].}
\end{figure}

Let's say that the algorithm ``fails" in step $i$ if in that step, you choose to contract an edge in the minimum cut (here we suppose that there is only one minimum cut). For all $u \in V$, let $d(u)$ be the degree of $u$.
\begin{align*} 
|E(S,\bar{S})| &\leq \displaystyle\min_u d(u) \leq \frac{1}{n} \sum_{u \in V} d(u) \leq \frac{2 \cdot m}{n} \\
\frac{|E(S,\bar{S})|}{m} &\leq \frac{2}{n}
\end{align*}
We fail in the first step if we pick an edge in the minimum cut, an event that occurs with probability $\displaystyle\frac{|E(S,\bar{S})|}{m}$.

\begin{align*} 
\mathbb{P}(\text{fail in $1^{st}$ step}) &\leq \frac{2}{n} \\
\mathbb{P}(\text{fail in $2^{nd}$ step} \ | \ \text{success in $1^{st}$ step}) &\leq \frac{2}{n-1} \\
& \vdots \\
\mathbb{P}(\text{fail in $i^{th}$ step} \ | \ \text{success till $(i-1)^{th}$ step})  &\leq \frac{2}{n-i+1}
\end{align*}
Let $Z_i$ be the event that the algorithm succeeds in step $i$. Thus, we have
\begin{align*}
\mathbb{P}(Z_1) &\geq \frac{n-2}{n} \\
\mathbb{P}(Z_2 | Z_1) &\geq \frac{n-3}{n-1} \\
& \vdots \\
\mathbb{P}(Z_i | Z_{i-1}  \cap \dots \cap Z_1)  &\geq \frac{n-i-1}{n-i+1} 
\end{align*}

Therefore:
\begin{align*} 
\mathbb{P}(\text{Success}) &= \mathbb{P}(Z_1 \cap Z_2 \cap \dots \cap Z_{n-2}) \\
&= \mathbb{P}(Z_1) \cdot \mathbb{P}(Z_2 | Z_1) \cdot \dots \cdot \mathbb{P}(Z_{n-2} | Z_1 \cap Z_2 \cap \dots \cap Z_{n-3}) \\
&\geq  \frac{n-2}{n} \cdot \frac{n-3}{n-1} \cdot \frac{n-4}{n-2} \cdot \frac{n-5}{n-3} \cdot ... \cdot \frac{2}{4} \cdot \frac{1}{3} \\
&= \frac{1}{n} \cdot \frac{1}{n-1} \cdot \frac{2}{1} \cdot \frac{1}{1} \\
&= \frac{2}{n(n-1)}\\
& \geq \frac{2}{n^2}
\end{align*}

If $n$ is too large, this is a very poor guarantee. So what we do instead, is run the algorithm several times and choose the best min-cut we find. If we run this algorithm $k$ times, we then have:

$$\mathbb{P}(\text{Success}) \geq 1 - \left(1- \frac{2}{n^2}\right)^k $$
A classical inequality yields $(1 - z)^{\frac{1}{z}} \leq \frac{1}{e}$. We hence choose to run the algorithm $k$ times, with $k =  \frac{n^2}{2}\log{\frac{1}{\delta}}$. We get:

$$\mathbb{P}(\text{Success}) \geq 1- \left( \frac{1}{e} \right) ^{\log{\frac{1}{\delta}}} = 1 - \delta $$
After running all these $k$ trials, we choose the one with minimum cut value.\\\\
At every step of the algorithm, we have to contract an edge. This takes at most $\mathcal{O}(n)$ time. We do at most $n$ such contractions. Thus for every run, we need $\mathcal{O}(n^2)$ time. We can choose $\delta$ to be, say $1/4$, and thus $k =  \frac{n^2}{2}\log{4}$ and this will ensure that within $\mathcal{O}(n^4)$ time the algorithm succeeds with probability at least $3/4$.


\section{Types of randomized algorithms}

\textbf{Las Vegas Algorithm (LV):} 
\begin{itemize}
	\item Always correct
	\item Runtime is random (small time with good probability)
	\item Examples : Quicksort, Hashing
\end{itemize}
\textbf{Monte Carlo Algorithm (MC):}
\begin{itemize}
	\item Always bounded in runtime
	\item Correctness is random
	\item Examples : Karger's min-cut algorithm
\end{itemize}
It's easy to see the following:
\begin{enumerate}
\item LV $\implies$ MC. Fix a time $T$. If the algorithm terminates before $T$, we output the answer, otherwise we output 1.
\item MC does not always imply LV. The implications holds when verifying a solution can be done much faster than finding one. In that case, we test the output of MC algorithm and stop only when a correct solution is found.
\end{enumerate}

\section{Back to Min-Cut : Can we do better?}

Initial stages of the algorithm are very likely to be correct. In particular, the first step is wrong with probability at most $2/n$. As we contract more edges, failure probability goes up. Moreover, earlier stages take more time compared to later ones. 

\textit{Idea:} Let us redistribute our iterations. Since earlier ones are more accurate and slower, why not do less of them at the beginning, and increasingly more as the number of edges decreases? \\

Let us formalize this idea. After t steps:

$\mathbb{P}(Success) \geq \dfrac{n-2}{n} \cdot \dfrac{n-3}{n-1} \cdot \dfrac{n-3}{n-1} \cdots \dfrac{n-t-1}{n-t+1} \approx \dfrac{(n-t)^2}{n^2}
$\\\

We equate this to $\frac{1}{2}$ to get $t = n - \frac{n}{\sqrt{2}}$. Thus, we have the following algorithm:

\begin{algorithm}[H]
	\caption{Getting better}
	\label{Algo2}
	\begin{algorithmic}[1]
    \Statex
    \Function{Algo2}{}
    \State Repeat Twice:
    	\State \quad Run contraction from $n \rightarrow n/\sqrt{2}$
        \State \quad Recursively call Algo2 on $n/\sqrt{2}$ set
        \State \Return the last two vertices for the best cut among the two
    \EndFunction
  \end{algorithmic}
\end{algorithm}

Runtime analysis:
\begin{align*}
T(n) &= 2 \left( \mathcal{O}(n^2) + T\left( \frac{n}{\sqrt{2}} \right) \right) \\
&= 2cn^2 + 2\cdot 2\cdot c\cdot \left( \frac{n}{\sqrt{2}} \right)^2 + \cdots \\
&= \mathcal{O}(n^2 \log n)
\end{align*}


Success Probability:
\begin{align*}
p(n) &= 1 - (1 - success \ in \ one \ branch)^2 \\
&= 1 - \left(1 - \frac{1}{2} \ p\left( \frac{n}{\sqrt{2}}\right) \right)^2 \\
&= 1 - \left(1 - \frac{1}{2} \ p\left( \frac{n}{\sqrt{2}}\right) \right)^2 \\
&= p\left( \frac{n}{\sqrt{2}}\right) - \frac{1}{4} \ p\left( \frac{n}{\sqrt{2}}\right)^2
\end{align*}

To solve this recursion, we let $x = \log_{\sqrt{2}}{n}$. Thus, setting $f(x) = p(n)$, we get
$$f(x) = f(x-1) - f(x-1)^2$$

We observe that setting $f(x) = \frac{1}{x}$ gives:
$$f(x-1) - f(x) = \frac{1}{x-1} - \frac{1}{x} = \frac{1}{x(x-1)} \approx \frac{1}{(x-1)^2} = f(x-1)^2$$

Hence, we have $p(n) = \mathcal{O} \left( \frac{1}{\log n} \right) $. Thus, to get $3/4$ probability of success, we need $\mathcal{O}(n^2 \log^2 n)$ time (which is much better than before, so we're happy).









\newpage
\section*{References}
[1] Thore Husfeldt. ``Single run of Karger's Mincut algorithm."
     \textit{Wikipedia}, N.p., 12 September 2012. Web.
     \url{https://fr.wikipedia.org/wiki/Algorithme_de_Karger#/media/File:Single_run_of_Karger%E2%80%99s_Mincut_algorithm.svg}


\end{document}