slides: architecture, load balancing, redundancy
parent
4bdd69ead9
commit
f2280de2ba
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,115 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
id="svg8"
|
||||
version="1.1"
|
||||
viewBox="0 0 75.179466 73.22541"
|
||||
height="73.22541mm"
|
||||
width="75.179466mm">
|
||||
<defs
|
||||
id="defs2" />
|
||||
<metadata
|
||||
id="metadata5">
|
||||
<rdf:RDF>
|
||||
<cc:Work
|
||||
rdf:about="">
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title></dc:title>
|
||||
</cc:Work>
|
||||
</rdf:RDF>
|
||||
</metadata>
|
||||
<g
|
||||
transform="translate(-78.05208,-148.9788)"
|
||||
id="layer1">
|
||||
<circle
|
||||
r="36.380207"
|
||||
cy="185.59151"
|
||||
cx="114.99925"
|
||||
id="path815"
|
||||
style="opacity:1;fill:none;fill-opacity:0.8956522;fill-rule:nonzero;stroke:#000000;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="153.44466"
|
||||
cx="98.12262"
|
||||
id="path817"
|
||||
style="opacity:1;fill:#19c21b;fill-opacity:0.82826089;fill-rule:nonzero;stroke:#000000;stroke-width:1.16499996;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="217.625"
|
||||
cx="132.70744"
|
||||
id="path817-4"
|
||||
style="opacity:1;fill:#19c21b;fill-opacity:0.82826089;fill-rule:nonzero;stroke:none;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="168.41249"
|
||||
cx="146.80594"
|
||||
id="path817-5"
|
||||
style="opacity:1;fill:#19c21b;fill-opacity:0.82826089;fill-rule:nonzero;stroke:none;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="199.48215"
|
||||
cx="81.07589"
|
||||
id="path817-6"
|
||||
style="opacity:1;fill:#19c21b;fill-opacity:0.82826089;fill-rule:nonzero;stroke:none;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="152.00833"
|
||||
cx="128.17174"
|
||||
id="path817-3"
|
||||
style="opacity:1;fill:#192cc2;fill-opacity:0.82826089;fill-rule:nonzero;stroke:none;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="218.38095"
|
||||
cx="99.974701"
|
||||
id="path817-3-7"
|
||||
style="opacity:1;fill:#192cc2;fill-opacity:0.82826089;fill-rule:nonzero;stroke:none;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="171.66309"
|
||||
cx="81.189285"
|
||||
id="path817-3-5"
|
||||
style="opacity:1;fill:#192cc2;fill-opacity:0.82826089;fill-rule:nonzero;stroke:none;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<circle
|
||||
r="3.0238097"
|
||||
cy="195.92917"
|
||||
cx="150.20773"
|
||||
id="path817-3-9"
|
||||
style="opacity:1;fill:#192cc2;fill-opacity:0.82826089;fill-rule:nonzero;stroke:none;stroke-width:0.46499997;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
|
||||
<text
|
||||
id="text887"
|
||||
y="161.34431"
|
||||
x="94.229462"
|
||||
style="color:#000000;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:3.52777767px;line-height:125%;font-family:'Source Sans Pro';-inkscape-font-specification:Sans;letter-spacing:0px;word-spacing:0px;display:inline;overflow:visible;visibility:visible;fill:#0b2817;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker:none;enable-background:accumulate"
|
||||
xml:space="preserve"><tspan
|
||||
style="font-size:4.93888903px;stroke-width:0.26458332px"
|
||||
y="161.34431"
|
||||
x="94.229462"
|
||||
id="tspan885">original</tspan></text>
|
||||
<text
|
||||
id="text891"
|
||||
y="170.07565"
|
||||
x="95.816963"
|
||||
style="color:#000000;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:3.52777767px;line-height:125%;font-family:'Source Sans Pro';-inkscape-font-specification:Sans;letter-spacing:0px;word-spacing:0px;display:inline;overflow:visible;visibility:visible;fill:#19c21b;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker:none;enable-background:accumulate"
|
||||
xml:space="preserve"><tspan
|
||||
style="font-style:normal;font-variant:normal;font-weight:600;font-stretch:normal;font-size:4.93888903px;font-family:'Source Sans Pro';-inkscape-font-specification:'Source Sans Pro, Semi-Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#19c21b;fill-opacity:1;stroke-width:0.26458332px"
|
||||
y="170.07565"
|
||||
x="95.816963"
|
||||
id="tspan889">first redundancy level</tspan></text>
|
||||
<text
|
||||
id="text891-7"
|
||||
y="197.12747"
|
||||
x="93.89312"
|
||||
style="color:#000000;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:3.52777767px;line-height:125%;font-family:'Source Sans Pro';-inkscape-font-specification:Sans;letter-spacing:0px;word-spacing:0px;display:inline;overflow:visible;visibility:visible;fill:#192cc2;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.26458332px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker:none;enable-background:accumulate"
|
||||
xml:space="preserve"><tspan
|
||||
style="font-style:normal;font-variant:normal;font-weight:600;font-stretch:normal;font-size:4.93888903px;font-family:'Source Sans Pro';-inkscape-font-specification:'Source Sans Pro, Semi-Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#192cc2;fill-opacity:1;stroke-width:0.26458332px"
|
||||
y="197.12747"
|
||||
x="93.89312"
|
||||
id="tspan889-6">second redundancy level</tspan></text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.7 KiB |
11
shell.nix
11
shell.nix
|
|
@ -29,6 +29,9 @@ mkShell {
|
|||
logreq
|
||||
koma-script
|
||||
xits
|
||||
stix
|
||||
stix2-otf
|
||||
stix2-type1
|
||||
opensans
|
||||
;
|
||||
|
||||
|
|
@ -38,9 +41,9 @@ mkShell {
|
|||
};
|
||||
})
|
||||
biber
|
||||
opensans-ttf
|
||||
xits-math
|
||||
stix-otf
|
||||
#opensans-ttf
|
||||
#xits-math
|
||||
#stix-otf
|
||||
fontconfig
|
||||
|
||||
# for instance DNS query script
|
||||
|
|
@ -63,5 +66,5 @@ mkShell {
|
|||
];
|
||||
|
||||
# for font discovery
|
||||
FONTCONFIG_FILE = makeFontsConf { fontDirectories = [ lmodern opensans-ttf xits-math stix-otf ]; };
|
||||
FONTCONFIG_FILE = makeFontsConf { fontDirectories = [ lmodern texlive.opensans.pkgs texlive.stix.pkgs ]; };
|
||||
}
|
||||
|
|
|
|||
119
talk-slides.tex
119
talk-slides.tex
|
|
@ -44,6 +44,7 @@
|
|||
\usepackage[backend=biber, sorting=none]{biblatex}
|
||||
|
||||
\usepackage{ccicons}
|
||||
\usepackage{wrapfig}
|
||||
|
||||
\usepackage{ifluatex}
|
||||
|
||||
|
|
@ -298,14 +299,124 @@ Hashtags are used for marking posts about certain topics or events:
|
|||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{System Architecture}{adding a P2P backend to the fediverse}
|
||||
\begin{frame}{System Architecture}{adding a DHT backend to the fediverse}
|
||||
|
||||
core idea: distributing responsibility for tags using a \textbf{D}istributed \textbf{H}ash \textbf{T}able \note{distribute responsibility for posts of a hashtag = relaying \& storage}
|
||||
core idea: distribute responsibility for tags among instances using a \textbf{D}istributed \textbf{H}ash \textbf{T}able, \note{distribute responsibility for posts of a hashtag = relaying \& storage}
|
||||
|
||||
based on Chord
|
||||
|
||||
\note[item]{DHT: structured P2P networks providing efficient (log N) key-value storage and lookup}
|
||||
\note[item]{self-organising, no central authority}
|
||||
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{System Architecture}{adding a DHT backend to the fediverse}
|
||||
\begin{columns}
|
||||
\begin{column}{0.4\textwidth}
|
||||
\begin{itemize}
|
||||
\item calculate hash value of keys and node IDs
|
||||
\item place these hashes onto the same circular name space
|
||||
\item each node keeps routing table of \(\log \#number\_nodes\) entries\note[item]{joining and leaving covered in paper}
|
||||
\end{itemize}
|
||||
\end{column}
|
||||
|
||||
\begin{column}{0.6\textwidth}
|
||||
\includegraphics[height=0.62\paperheight]{figures/finger_table_routing_1.pdf}
|
||||
\end{column}
|
||||
\end{columns}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{System Architecture}{adding a DHT backend to the fediverse}
|
||||
\begin{columns}
|
||||
\begin{column}{0.55\textwidth}
|
||||
\begin{itemize}
|
||||
\item next nodeID \(\geq\) \texttt{hash(hashtag)} (mod keyspace size) is responsible for handling posts containing \texttt{hashtag}
|
||||
\item DHT used for iterative lookup of responsible relay/ storage node
|
||||
\end{itemize}
|
||||
\end{column}
|
||||
|
||||
\begin{column}{0.45\textwidth}
|
||||
\includegraphics[height=0.62\paperheight]{figures/finger_table_routing_2.pdf}
|
||||
\end{column}
|
||||
\end{columns}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Publishing, Relaying and Storage}{lifecycle of posts}
|
||||
\begin{enumerate}
|
||||
\item publishing instance looks up responsible relay instance on DHT for each included hashtag
|
||||
\item publishing instance sends post to responsible relay instance
|
||||
\item relay instance looks up responsible storage node on DHT
|
||||
\item relay instance verifies incoming post's signature, then relays post URI (ID) to all subscribers + storage node\note[item]{only post ID relayed, but not full post content. Reasons: LDSignatures not supported everywhere, deniability \& revocation}
|
||||
\item subscribing instances can now retrieve the full authenticated post from received post URI
|
||||
\end{enumerate}
|
||||
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Publishing, Relaying and Storage}
|
||||
\begin{itemize}
|
||||
\item separate DHTs for relay and storage instances
|
||||
\item all actions after DHT lookup supposed to be done using ActivityPub via HTTPS
|
||||
\item subscription to hashtags/ querying posts is done at the responsible instance
|
||||
\end{itemize}
|
||||
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\note{so far so easy. But load distribution issues}
|
||||
\begin{itemize}
|
||||
\item node ID determines set of hashtags handled by instance
|
||||
\item problem: for security reasons, node \textbf{must not} choose their IDs freely
|
||||
\item Can instances be overloaded by their assigned hashtag posts?
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Distribution of Posts per Tag}
|
||||
\includegraphics[width=0.49\textwidth]{statistics/twitter_hashtags_total.png}
|
||||
~
|
||||
\includegraphics[width=0.49\textwidth]{statistics/geraspora_hashtags_total.png}
|
||||
|
||||
\note{analysis of a 1 month dump of Twitter, Geraspora (Diaspora) and Friendica posts\\
|
||||
Twitter: 70\% of posts used just once\\
|
||||
note the logarithmic axis!}
|
||||
|
||||
distribution of posts per hashtag follows a steep power law
|
||||
|
||||
\note{So what if a small node gets several large hashtags? => need for load balancing}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Load Balancing}{of hashtags between nodes}
|
||||
\begin{itemize}
|
||||
\item \textit{k-choices} algorithm by Ledlie and Seltzer
|
||||
\item each node can choose from \(\kappa\) possible IDs
|
||||
\item nodes have a \textbf{capacity} and choose set of active IDs according to lowest mismatch of own and neighbour node capacity
|
||||
\item querying load of potential IDs before joining, periodic re-balancing
|
||||
\item independent load balancing of relay and storage nodes due to independent DHTs
|
||||
\end{itemize}
|
||||
% for Kolloquium, add simulation result
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Redundancy}
|
||||
\begin{columns}
|
||||
\begin{column}{0.55\textwidth}
|
||||
\begin{itemize}
|
||||
\item store redundant copies of hashtag data at equal distances on Chord ring\note{resilience against node failure, allows data validation through cross-checking}
|
||||
\item default redundancy: \(2^2 = 4\), scalable in powers of 2
|
||||
\item \textbf{relay nodes}: hot standby nodes take over in overload situations (load spikes)
|
||||
\item \textbf{storage nodes}: overloaded nodes can split stored posts by content hash and double redundancy set
|
||||
\end{itemize}
|
||||
\end{column}
|
||||
|
||||
\begin{column}{0.45\textwidth}
|
||||
\includegraphics[width=\textwidth]{figures/redundancy_ring.pdf}
|
||||
\end{column}
|
||||
\end{columns}
|
||||
\end{frame}
|
||||
|
||||
|
||||
|
||||
why even still use classic push federation?
|
||||
|
||||
|
||||
\section{Discussion}
|
||||
|
||||
\begin{frame}{Discussion}{I need YOUR feedback}
|
||||
|
|
@ -322,6 +433,10 @@ CloudFlare
|
|||
|
||||
load and capacity factor
|
||||
|
||||
performance: batching, exponential back-off, no relayable sigs
|
||||
|
||||
security: node ID derivation scheme
|
||||
|
||||
\section{Summary}
|
||||
|
||||
\begin{frame}{Summary}
|
||||
|
|
|
|||
Loading…
Reference in New Issue