commit 1dde2d697bd957beb29772de4df0f2312e5c16a3 Author: finga Date: Mon Sep 19 14:27:29 2022 +0200 Create the presentation diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c435426 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +svg-inkscape/ +prometheus-exporter-rs.html +prometheus-exporter-rs.pdf +prometheus-exporter-rs.pyg +prometheus-exporter-rs.tex diff --git a/img/Bruine_roest_op_tarwe_(Puccinia_recondita_f.sp._tritici_on_Triticum_aestivum).jpg b/img/Bruine_roest_op_tarwe_(Puccinia_recondita_f.sp._tritici_on_Triticum_aestivum).jpg new file mode 100644 index 0000000..786c719 Binary files /dev/null and b/img/Bruine_roest_op_tarwe_(Puccinia_recondita_f.sp._tritici_on_Triticum_aestivum).jpg differ diff --git a/img/cargo.png b/img/cargo.png new file mode 100644 index 0000000..293e76f Binary files /dev/null and b/img/cargo.png differ diff --git a/img/compiler_complaint.png b/img/compiler_complaint.png new file mode 100644 index 0000000..10ac5eb Binary files /dev/null and b/img/compiler_complaint.png differ diff --git a/img/prometheus-icon-color.svg b/img/prometheus-icon-color.svg new file mode 100644 index 0000000..4e3ad04 --- /dev/null +++ b/img/prometheus-icon-color.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/img/rust-logo-blk.svg b/img/rust-logo-blk.svg new file mode 100644 index 0000000..1a6c762 --- /dev/null +++ b/img/rust-logo-blk.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/prometheus-exporter-rs.org b/prometheus-exporter-rs.org new file mode 100644 index 0000000..8aed4ef --- /dev/null +++ b/prometheus-exporter-rs.org @@ -0,0 +1,716 @@ +#+STARTUP: beamer +#+OPTIONS: ':nil *:t -:t ::t <:t H:3 \n:nil ^:nil arch:headline +#+OPTIONS: author:t broken-links:nil c:nil creator:nil +#+OPTIONS: d:(not "LOGBOOK") date:t e:t email:nil f:t inline:t num:t +#+OPTIONS: p:nil pri:nil prop:nil stat:t tags:t tasks:t tex:t +#+OPTIONS: timestamp:t title:t toc:t todo:t |:t +#+TITLE: @@latex:\includesvg[height=.25\textheight]{img/prometheus-icon-color}\hspace{1cm}\includesvg[height=.25\textheight]{img/rust-logo-blk}\newline @@Create a simple Prometheus Exporter with Rust +#+SUBTITLE: From scratch to Debian package. +#+DATE: \today +#+AUTHOR: [[mailto:finga@onders.org][finga]] +#+EMAIL: finga@onders.org +#+DESCRIPTION: Create a simple Prometheus Exporter with Rust, from scratch to Debian package. +#+LANGUAGE: en +#+KEYWORDS: prometheus rust programming +#+SELECT_TAGS: export +#+EXCLUDE_TAGS: noexport +#+CREATOR: Emacs 26.1 (Org mode 9.1.9) + +#+OPTIONS: H:2 +#+LATEX_CLASS: beamer +#+LATEX_CLASS_OPTIONS: [aspectratio=1610] +#+COLUMNS: %45ITEM %10BEAMER_env(Env) %10BEAMER_act(Act) %4BEAMER_col(Col) %8BEAMER_opt(Opt) +#+LATEX_HEADER: \usepackage{svg}\hypersetup{colorlinks=true,linkcolor=black,urlcolor=gray} +#+BEAMER_THEME: Frankfurt +#+BEAMER_COLOR_THEME: seagull +#+BEAMER_FONT_THEME: +#+BEAMER_INNER_THEME: +#+BEAMER_OUTER_THEME: +#+BEAMER_HEADER: \institute[INST]{\href{https://onders.org}{onders.org}} + +* Objective + +** Objective + +*** Abstract + Acquire, (process) and serve metrics. + +*** What we are actually going to do + Parse ~/proc/loadavg~ periodically and serve its latest values. + +*** Example content of ~/proc/loadavg~ + #+begin_src +0.13 0.14 0.09 1/273 3160 + #+end_src + +** Create a new Project + +*** New Cargo Project + To begin we create a new Cargo project. + #+begin_src sh +$ cargo new loadavg-exporter + #+end_src + +** Project settings + Add meta data to ~Cargo.toml~. + +*** ~Cargo.toml~ + @@latex:\scriptsize@@ + #+begin_src toml +[package] +name = "loadavg-exporter" +version = "0.1.0" +authors = ["finga "] +edition = "2021" +description = "Prometheus exporter example to export the load average." +readme = "README.md" +license = "GPL-3.0-or-later" +repository = "https://git.onders.org/finga/loadavg-exporter.git" +keywords = ["prometheus", "metrics"] +categories = ["command-line-utilities"] + #+end_src + +** Read from ~/proc/loadavg~ +*** ~src/main.rs~: Print content of ~/proc/loadavg~ + @@latex:\scriptsize@@ + #+begin_src rust +use std::{ + fs::File, + io::{BufRead, BufReader}, + path::Path, +}; + +fn parse_loadavg

(filename: P) +where + P: AsRef, +{ + let file = File::open(&filename).unwrap(); + for line in BufReader::new(file).lines() { + println!("{}", line.unwrap()); + } +} + +fn main() { + parse_loadavg("/proc/loadavg") +} + #+end_src + +** Use of =Result= + +*** It is "just" an enum + @@latex:\scriptsize@@ + #+begin_src rust +enum Result { + Ok(T), + Err(E), +} + #+end_src + +*** How to use =Result= +- ~Results~ "must" be used +- There is the question mark operator: =?= +- =?= can only be used in functions that return =Result= +- For more see the docs[fn:result] +- Historically the =?= operator replaces the =try!=[fn:try] macro + +** Set some ~clippy~ settings + +*** Create ~.cargo/config~ + @@latex:\scriptsize@@ +#+begin_src +[target.'cfg(feature = "cargo-clippy")'] +rustflags = [ + "-Dwarnings", + "-Dclippy::pedantic", + "-Dclippy::nursery", + "-Dclippy::cargo", +] +#+end_src + +** Use ~anyhow~ + +*** Add dependency to ~Cargo.toml~ + @@latex:\scriptsize@@ + #+begin_src toml +[dependencies] +anyhow = "1" + #+end_src + +*** Use ~anyhow~ where suitable + @@latex:\scriptsize@@ + #+begin_src diff ++use anyhow::Result; + ... +-fn parse_loadavg

(filename: P) ++fn parse_loadavg

(filename: P) -> Result<()> + ... +- let file = File::open(&filename).unwrap(); ++ let file = File::open(&filename)?; + for line in BufReader::new(file).lines() { +- println!("{}", line.unwrap()); ++ println!("{}", line?); + } ++ Ok(()) + } +... + #+end_src + +** Create an atomic for =f64= + +*** Create an atomic =f64= type + @@latex:\scriptsize@@ + #+begin_src rust +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Default)] +struct AtomicF64 { + storage: AtomicU64, +} + +impl AtomicF64 { + fn store(&self, value: f64, ordering: Ordering) { + let as_u64 = value.to_bits(); + self.storage.store(as_u64, ordering); + } + + fn load(&self, ordering: Ordering) -> f64 { + let as_u64 = self.storage.load(ordering); + f64::from_bits(as_u64) + } +} + #+end_src + +** What does the =#[derive(Default)]= attribute do? + This calls a procedural derive macro[fn:derive] to generate =AtomicF64::default()=. + +*** What is generated? + @@latex:\scriptsize@@ + #+begin_src rust + #[automatically_derived] + #[allow(unused_qualifications)] + impl ::core::default::Default for AtomicF64 { + #[inline] + fn default() -> AtomicF64 { + AtomicF64 { + storage: ::core::default::Default::default(), + } + } + } + #+end_src + +** Create a storage for the metrics + +*** Create a struct to keep the metrics + @@latex:\scriptsize@@ + #+begin_src rust +#[derive(Default)] +struct Metrics { + load_1: AtomicF64, + load_5: AtomicF64, + load_15: AtomicF64, +} + #+end_src + +** Introduce logging + +*** How to configure logging from the outside? + - The =RUST_LOG= environment variable is the answer + - Use =RUST_LOG=trace cargo r= to set the log level to ~trace~ and + run the project + - Use =RUST_LOG=error,path::to::module=trace= to set the overall + log level to ~error~ and the log level of =path::to::module= to + ~trace~ + +*** Add new dependencies in ~Cargo.toml~ + @@latex:\scriptsize@@ + #+begin_src diff + [dependencies] + anyhow = "1" ++log = "0.4" ++env_logger = "0.9" + #+end_src + +** Introduce logging + +*** Generate some log messages + @@latex:\scriptsize@@ + #+begin_src diff + use anyhow::Result; ++use log::{debug, info}; + use std::{ + fs::File, + io::{BufRead, BufReader}, +... + fn parse_loadavg

(filename: P) -> Result<()> + where +- P: AsRef, ++ P: AsRef + std::fmt::Display, + { ++ debug!("Read load average from {}", filename); + let file = File::open(&filename)?; +... + fn main() -> Result<()> { ++ env_logger::init(); ++ ++ info!("{} started", env!("CARGO_PKG_NAME")); + parse_loadavg("/proc/loadavg")?; + #+end_src + +** Parse the ~loadavg~ line + +*** Parse the fields + @@latex:\scriptsize@@ + #+begin_src diff +-fn parse_loadavg

(filename: P) -> Result<()> ++fn parse_loadavg

(filename: P, metrics: Arc) -> Result<()> + ... + let file = File::open(&filename)?; +- for line in BufReader::new(file).lines() { +- println!("{}", line?); ++ let mut data = String::new(); ++ ++ BufReader::new(file).read_to_string(&mut data)?; ++ let data = data.trim(); ++ trace!("Data to parse: {}", data); ++ let fields: Vec<&str> = data.split(' ').collect(); ++ ++ if fields.len() != 5 { ++ bail!( ++ "Expected to read 5 space separated fields from {}", ++ filename ++ ); + } + ... + #+end_src + +** Store the parsed values in our =Metrics= struct + +*** Before returning we store the gathered data + @@latex:\scriptsize@@ + #+begin_src diff + fn parse_loadavg

(filename: P, metrics: Arc) -> Result<()> + where + P: AsRef + std::fmt::Display, + { + ... ++ trace!("Parsed fields: {:?}", fields); ++ ++ metrics ++ .load_1 ++ .store(fields[0].parse::()?, Ordering::Relaxed); ++ metrics ++ .load_5 ++ .store(fields[1].parse::()?, Ordering::Relaxed); ++ metrics ++ .load_15 ++ .store(fields[2].parse::()?, Ordering::Relaxed); ++ + Ok(()) + } + #+end_src + +** Introduce =tokio=[fn:tokio] + @@latex:\scriptsize@@ +*** Add =tokio= to =Cargo.toml= +#+begin_src diff + ... ++tokio = { version = "1", features = ["macros", "time", "rt-multi-thread"] } +#+end_src + +*** Add =use= statements to bring utilities into scope +#+begin_src diff + use anyhow::{bail, Result}; + use log::{debug, info, trace}; + use std::{ + fs::File, + io::{BufReader, Read}, + path::Path, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, ++ time::Duration, + }; ++use tokio::time::sleep; +#+end_src + +** Periodically poll the data from =/proc/loadavg= + +*** Call the loadavg parser in an infinite loop and sleep + @@latex:\scriptsize@@ + #+begin_src rust +async fn poll_loadavg

(filename: P, interval: u64, metrics: Arc) -> Result<()> +where + P: AsRef + std::fmt::Display, +{ + debug!("Reading {} every {} seconds", filename, interval); + + loop { + trace!("Polling loadavg from {}", filename); + parse_loadavg(&filename, metrics.clone())?; + sleep(Duration::from_secs(interval)).await; + } +} + #+end_src + +** Make =main= use =tokio= + +*** And adopt our =main()= to use it + @@latex:\scriptsize@@ + #+begin_src diff +-fn main() -> Result<()> { ++#[tokio::main] ++async fn main() -> Result<()> { + ... +- parse_loadavg("/proc/loadavg", Arc::clone(&metrics))?; ++ poll_loadavg("/proc/loadavg", 5, Arc::clone(&metrics)).await?; + #+end_src + +** What is =#[tokio::main]=? + +*** It is syntactic sugar as following: + @@latex:\scriptsize@@ +#+begin_src rust +#[tokio::main] +async fn main() { + println!("hello"); +} +#+end_src + +*** ... just generates this: + @@latex:\scriptsize@@ +#+begin_src rust +fn main() { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + println!("hello"); + }) +} +#+end_src + +** Serve the metrics data + +*** Add dependencies to =Cargo.toml=... + @@latex:\scriptsize@@ + #+begin_src diff + [dependencies] + ... ++axum = "0.5" + ... + #+end_src + +*** and import further necessities in our program... + @@latex:\scriptsize@@ + #+begin_src diff + use anyhow::{bail, Result}; ++use axum::{routing::get, Extension, Router, Server}; + use log::{debug, info, trace}; + use std::{ + fs::File, + io::{BufReader, Read}, ++ net::{IpAddr, Ipv4Addr, SocketAddr}, + path::Path, + ... + }; +-use tokio::time::sleep; ++use tokio::{spawn, time::sleep, try_join}; + #+end_src + +** Generate the response string + +*** Create the served metrics data + @@latex:\scriptsize@@ + #+begin_src rust +#[allow(clippy::unused_async)] +async fn serve_metrics(Extension(metrics): Extension>) -> String { + format!( + r"# HELP System load (1m). +# TYPE load_1 gauge +load_1 {} +# HELP System load (5m). +# TYPE load_5 gauge +load_5 {} +# HELP System load (15m). +# TYPE load_15 gauge +load_15 {} +", + metrics.load_1.load(Ordering::Relaxed), + metrics.load_5.load(Ordering::Relaxed), + metrics.load_15.load(Ordering::Relaxed), + ) +} + #+end_src + +** Create a =Router= to route and serve the http endpoint + +*** Basically thats our http listener + @@latex:\scriptsize@@ + #+begin_src rust +async fn listen_http(address: IpAddr, port: u16, metrics: Arc) -> Result<()> { + let app = Router::new() + .route("/metrics", get(serve_metrics)) + .layer(Extension(metrics)); + let addr = SocketAddr::from((address, port)); + debug!("Listening on {}:{}", address, port); + Ok(Server::bind(&addr).serve(app.into_make_service()).await?) +} + #+end_src + +** Read from file and serve http asynchronously + +*** Poll loadavg, listen for http and return on error + @@latex:\scriptsize@@ + #+begin_src diff + #[tokio::main] + async fn main() -> Result<()> { + ... +- poll_loadavg("/proc/loadavg", 5, Arc::clone(&metrics)).await?; ++ ++ let (poller, listener) = try_join!( ++ spawn(poll_loadavg("/proc/loadavg", 5, Arc::clone(&metrics))), ++ spawn(listen_http( ++ IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), ++ 8000, ++ Arc::clone(&metrics), ++ )), ++ )?; ++ poller?; ++ listener?; + + Ok(()) + } + #+end_src + +** Early Conclusion + This concludes the first step of creating a small prometheus + exporter. + +*** Note + We could have created a more simple solution without using =tokio= + and by parsing ~/proc/loadavg~ on each request. Still, this shows + a bit of =tokio= and further might not appliciable when slightly + more complex requirements are needed. + +** The Finishing + Although this example already works here are some additional steps + to improve certain aspects. + +*** Further improvements + - Parse command line arguments with ~clap~: The Command Line + Argument Parser + - Use ~cargo-deb~ to generate a Debian Package + - Cross compile the project for other architectures + +* ~clap~ + +** Parse command line arguments with ~clap~ + +*** What is ~clap~ + - There are several ways of how ~clap~ can be used (derive, + builder, etc..) + - Documentation: [[https://docs.rs/clap/latest/clap/]] + - Examples: [[https://github.com/clap-rs/clap/tree/master/examples]] + +** Add build dependencies for ~clap~ + +*** Add ~clap~ dependencies to ~Cargo.toml~ + @@latex:\scriptsize@@ + #+begin_src diff + axum = "0.5" ++clap = { version = "3", features = ["derive"] } + #+end_src + +** Have a struct holding all arguments + +*** Create a struct which holds all arguments + @@latex:\scriptsize@@ + #+begin_src rust +#[derive(Parser)] +#[clap(about, version, author)] +struct Args { + /// The path to the file to parse the load average parameters. + #[clap(short, long, default_value = "/proc/loadavg")] + file: String, + /// The intervall how often the file is queried + #[clap(short, long, default_value = "10")] + interval: u64, + /// The IPv4 or IPv6 address where the metrics are served. + #[clap(short, long, default_value = "127.0.0.1")] + address: IpAddr, + /// The port where the metrics are served. + #[clap(short, long, default_value = "9111")] + port: u16, + /// Produce verbose output, multiple -v options increase the verbosity + #[clap(short, long, global = true, parse(from_occurrences))] + verbose: u8, +} + #+end_src + +** Make use of the new arguments + +*** Parse the arguments and set the log level accordingly + @@latex:\scriptsize@@ + #+begin_src diff + #[tokio::main] + async fn main() -> Result<()> { +- env_logger::init(); ++ let cli = Cli::parse(); ++ ++ env_logger::Builder::from_env(env_logger::Env::default().default_filter_or( ++ match cli.verbose { ++ 0 => "error", ++ 1 => "warn", ++ 2 => "info", ++ 3 => "debug", ++ _ => "trace", ++ }, ++ )) ++ .init(); + #+end_src + +** Make use of the new arguments + +*** Pass the argument values to the procedures + @@latex:\scriptsize@@ + #+begin_src diff + let (poller, listener) = try_join!( +- spawn(poll_loadavg("/proc/loadavg", 5, Arc::clone(&metrics))), +- spawn(listen_http( +- IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), +- 8000, +- Arc::clone(&metrics), +- )), ++ spawn(poll_loadavg(cli.file, cli.interval, Arc::clone(&metrics))), ++ spawn(listen_http(cli.address, cli.port, Arc::clone(&metrics))), + )?; + poller?; + listener?; + #+end_src + +* Debian Package + +** Build a Debian Package with the help of ~cargo-deb~ + +*** Create ~debian/service~ + @@latex:\scriptsize@@ + #+begin_src +[Unit] +Description=Load average Prometheus exporter + +[Service] +Restart=always +EnvironmentFile=/etc/default/loadavg-exporter +ExecStart=/usr/bin/loadavg-exporter $ARGS +Type=simple +ProtectSystem=strict +PrivateDevices=yes +PrivateUsers=yes +RestrictNamespaces=yes + +[Install] +WantedBy=multi-user.target + #+end_src + +** Build a Debian Package with the help of ~cargo-deb~ + +*** Create ~debian/default~ + @@latex:\scriptsize@@ + #+begin_src +ARGS="" +# loadavg-exporter supports the following options: +# -a, --address

The IPv4 or IPv6 address where the metrics are served +# [default: 127.0.0.1] +# -f, --file The path to the file to parse the load average parameters +# [default: /proc/loadavg] +# -i, --interval The intervall how often the file is queried [default: 10] +# -p, --port The port where the metrics are served [default: 9112] +# -v, --verbose Produce verbose output, multiple -v options increase the +# verbosity + #+end_src + +** Build a Debian Package with the help of ~cargo-deb~ + +*** Add the ~package.metadata.deb~ section to ~Cargo.toml~ + @@latex:\scriptsize@@ + #+begin_src toml +[package.metadata.deb] +extended-description = "Load average Prometheus exporter." +section = "utility" +maintainer-scripts = "debian/" +systemd-units = { enable = true } +assets = [ + ["target/release/loadavg-exporter", "usr/bin/", "755"], + ["debian/default", "etc/default/loadavg-exporter", "644"], +] + #+end_src + +* Cross compile + +** Cross compile for aarch64/arm64 +You need to have the right compiler dependencies installed, on Debian +that is: ~dummy~ + +*** Add targets to ~.cargo/config~ + @@latex:\scriptsize@@ + #+begin_src +[target.armv7-unknown-linux-gnueabihf] +linker = "arm-linux-gnueabihf-gcc" + +[target.aarch64-unknown-linux-gnu] +linker = "aarch64-linux-gnu-gcc" + #+end_src + +* Examples + +** Example 1: MightyOhm Geiger Counter +*** Abstract + A sensor connected to a system to log the sensor data. + +*** Technology + - [[https://mightyohm.com/blog/products/geiger-counter/][MightyOhm Geiger Counter]] + - [[https://www.raspberrypi.com/products/raspberry-pi-3-model-b-plus/][RaspberryPi 3 Model B+]] + - [[https://prometheus.io/][Prometheus]] + - [[https://grafana.com/][Grafana]] + - [[https://www.rust-lang.org/][Rust]] + +** Example 1: Sensor + +*** Reading Data from the Sensor + The Geiger Counter's serial port is connected to the UART port of + the Pi. So our program need to listen on the serial line to + receive the Data + +** Example 1: Prometheus +*** Prometheus Exporter + In order to pipe the Data to Prometheus we will open a port to + listen for incomming http requests to respond with the sensors + data. + +# ** Reading from the Serial Line +# To start with reading data from the serial line we use the +# ~serialport~ crate. So we do need to add it to the projects +# dependencies. +# *** ~Cargo.toml~ +# #+begin_src toml +# [dependencies] +# serialport = "4.1" +# #+end_src +# *** :B_ignoreheading: +# :PROPERTIES: +# :BEAMER_env: ignoreheading +# :END: +# And take a look at the ~seralport~ +# documentation[fn:serialport_docs] and a simple +# example[fn:serialport_example_receive] about how to read from the +# serial line. + +** Example 2: DHT11 Exporter + +* Footnotes +[fn:result] =Result= docs: [[https://doc.rust-lang.org/std/result/]] +[fn:try] =try!= docs: https://doc.rust-lang.org/std/macro.try.html +[fn:derive] [[https://doc.rust-lang.org/reference/procedural-macros.html#derive-macros][https://doc.rust-lang.org/reference/procedural-macros.html]] +[fn:tokio] https://tokio.rs/ + +# [fn:serialport_docs] [[https://docs.rs/serialport/latest/serialport/][~serialport~ docs]] +# [fn:serialport_example_receive] [[https://github.com/serialport/serialport-rs/blob/main/examples/receive_data.rs][~serialport-rs/examples/receive_data.rs~]]