From 5fb711bd853fd4dc3f2eb9b1e95adf8e6f7bff2f Mon Sep 17 00:00:00 2001 From: Paul Zinselmeyer Date: Thu, 25 Jan 2024 17:03:38 +0100 Subject: [PATCH] aead --- Cargo.lock | 53 ++++++ server/Cargo.toml | 3 + server/src/error.rs | 8 + server/src/main.rs | 379 ++++++++++++++++++++++++++++++++++++----- server/src/metadata.rs | 2 +- server/src/web_util.rs | 6 +- 6 files changed, 400 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd3f9cc..4858dc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -218,6 +218,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-extra" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "895ff42f72016617773af68fb90da2a9677d89c62338ec09162d4909d86fdd8f" +dependencies = [ + "axum 0.7.4", + "axum-core 0.4.3", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "serde", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-macros" version = "0.4.1" @@ -294,6 +317,7 @@ name = "bin" version = "0.1.0" dependencies = [ "axum 0.7.4", + "axum-extra", "axum-oidc", "bytes", "chacha20", @@ -306,6 +330,7 @@ dependencies = [ "log", "markdown", "pin-project-lite", + "poly1305", "prometheus-client", "rand", "render", @@ -320,6 +345,7 @@ dependencies = [ "tower", "tower-http", "tower-sessions", + "zeroize", ] [[package]] @@ -1652,6 +1678,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "open" version = "5.0.1" @@ -1853,6 +1885,17 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -2907,6 +2950,16 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" diff --git a/server/Cargo.toml b/server/Cargo.toml index 5f7d024..b0e54c1 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -32,6 +32,9 @@ sha3 = "0.10" hex = "0.4" bytes = "1.5" pin-project-lite = "0.2" +poly1305 = "0.8.0" +zeroize = "1.7.0" +axum-extra = { version="0.9.2", features=["async-read-body"]} reqwest = { version="0.11", default_features=false, features=["rustls-tls", "json"] } jsonwebtoken = "9.2.0" diff --git a/server/src/error.rs b/server/src/error.rs index ab2c589..831ecdc 100644 --- a/server/src/error.rs +++ b/server/src/error.rs @@ -21,6 +21,9 @@ pub enum Error { #[error("file exists")] DataFileExists, + #[error("bin verification failed")] + BinVerificationFailed, + #[error("hex error: {:?}", 0)] Hex(#[from] hex::FromHexError), @@ -59,6 +62,11 @@ impl IntoResponse for Error { fn into_response(self) -> axum::response::Response { debug!("{:?}", &self); match self { + Self::BinVerificationFailed => ( + StatusCode::INTERNAL_SERVER_ERROR, + "bin verification failed\n", + ) + .into_response(), Self::PhraseInvalid => (StatusCode::BAD_REQUEST, "url is not valid\n").into_response(), Self::BinNotFound => (StatusCode::NOT_FOUND, "bin does not exist\n").into_response(), Self::DataFileExists => { diff --git a/server/src/main.rs b/server/src/main.rs index 152e92f..247771e 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,10 +1,13 @@ #![deny(clippy::unwrap_used)] +use axum_extra::body::AsyncReadBody; use axum_oidc::{ error::{ExtractorError, MiddlewareError}, EmptyAdditionalClaims, OidcAuthLayer, OidcClaims, OidcLoginLayer, }; -use duration_str::{deserialize_option_duration, parse_std}; +use duration_str::parse_std; use jwt::JwtApplication; +use pin_project_lite::pin_project; +use poly1305::{universal_hash::UniversalHash, Poly1305, Tag}; use prometheus_client::{ encoding::EncodeLabelSet, metrics::{counter::Counter, family::Family, gauge::Gauge}, @@ -14,8 +17,11 @@ use sailfish::TemplateOnce; use std::{ borrow::Borrow, env, + io::{ErrorKind, SeekFrom}, + pin::Pin, str::FromStr, sync::Arc, + task::{ready, Context, Poll}, time::{Duration, SystemTime, UNIX_EPOCH}, }; use tower_http::services::ServeDir; @@ -33,39 +39,40 @@ use axum::{ HeaderMap, HeaderValue, Request, StatusCode, Uri, }, response::{Html, IntoResponse, Redirect, Response}, - routing::{delete, get, post}, - BoxError, Router, + routing::{delete, get}, + Router, }; use chacha20::{ - cipher::{KeyIvInit, StreamCipher}, + cipher::{generic_array::GenericArray, KeyInit, KeyIvInit, StreamCipher, StreamCipherSeek}, ChaCha20, }; use futures_util::StreamExt; use garbage_collector::GarbageCollector; -use log::{debug, warn}; +use log::{debug, info, warn}; use serde::Deserialize; -use sha3::{Digest, Sha3_256}; use tokio::{ fs::{self, File}, - io::{AsyncWriteExt, BufReader, BufWriter}, + io::{ + AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter, + ReadBuf, + }, net::TcpListener, }; use util::{IdSalt, KeySalt}; +use zeroize::Zeroize; use crate::{ error::Error, jwt::Claims, metadata::Metadata, util::{Id, Key, Nonce, Phrase}, - web_util::DecryptingStream, }; mod error; mod garbage_collector; mod jwt; mod metadata; mod util; -mod web_util; /// length of the "phrase" that is used to access the bin (https://example.com/) const PHRASE_LENGTH: usize = 16; @@ -242,7 +249,7 @@ async fn get_index( let metadata = Metadata { subject, nonce: nonce.to_hex(), - etag: None, + tag: None, size: None, content_type: None, expires_at, @@ -329,30 +336,25 @@ async fn upload_bin( if !path.exists() { let key = Key::from_phrase(&phrase, &app_state.key_salt); let nonce = Nonce::from_hex(&metadata.nonce)?; - let mut cipher = ChaCha20::new(key.borrow(), nonce.borrow()); let file = File::create(&path).await?; - let mut writer = BufWriter::new(file); - - let mut etag_hasher = Sha3_256::new(); - let mut size: u64 = 0; + let writer = BufWriter::new(file); let mut ttl = params .ttl - .map(|x| duration_str::parse(&x).map_err(|_| Error::InvalidTtl)) + .map(|x| duration_str::parse(x).map_err(|_| Error::InvalidTtl)) .transpose()? .unwrap_or(Duration::from_secs(24 * 3600)); + let mut writer = EncWriter::new(writer, key, nonce, &[]); + match data { MultipartOrStream::Stream(stream) => { let mut stream = stream.into_data_stream(); while let Some(chunk) = stream.next().await { - let mut buf = chunk.unwrap_or_default().to_vec(); - etag_hasher.update(&buf); - size += buf.len() as u64; - cipher.apply_keystream(&mut buf); - writer.write_all(&buf).await?; + writer.write_all(chunk.unwrap_or_default().as_ref()).await?; } + writer.shutdown().await?; metadata.content_type = match headers.get(CONTENT_TYPE) { Some(content_type) => Some( content_type @@ -373,11 +375,7 @@ async fn upload_bin( { if field.name().unwrap_or_default() == "file" && !file_read { while let Some(chunk) = field.chunk().await.unwrap_or_default() { - let mut buf = chunk.to_vec(); - etag_hasher.update(&buf); - size += buf.len() as u64; - cipher.apply_keystream(&mut buf); - writer.write_all(&buf).await?; + writer.write_all(chunk.as_ref()).await?; } metadata.content_type = Some( field @@ -398,6 +396,9 @@ async fn upload_bin( } } writer.flush().await?; + writer.shutdown().await?; + + let tag = writer.tag().expect("valid tag"); if let Some(expires_at) = SystemTime::now() .duration_since(UNIX_EPOCH)? @@ -407,8 +408,8 @@ async fn upload_bin( metadata.expires_at = expires_at; } - metadata.etag = Some(hex::encode(etag_hasher.finalize())); - metadata.size = Some(size); + metadata.tag = Some(hex::encode(tag)); + metadata.size = Some(writer.size() as u64); metadata.to_file(&metadata_path).await?; @@ -459,15 +460,41 @@ async fn get_item( } else { //TODO(pfz4): Maybe add link handling let file = File::open(&path).await?; - let reader = BufReader::new(file); + let mut reader = BufReader::new(file); - let body = Body::from_stream(DecryptingStream::new( - reader, - id.clone(), - &metadata, - &key, - &nonce, - )); + let tag = *Tag::from_slice( + &hex::decode(metadata.tag.as_deref().unwrap_or("")).unwrap_or_default(), + ); + + { + let mut cipher = ChaCha20::new(key.borrow(), nonce.borrow()); + let mut mac_key = poly1305::Key::default(); + cipher.apply_keystream(&mut mac_key); + + let mut mac = Poly1305::new(GenericArray::from_slice(&mac_key)); + mac_key.zeroize(); + + mac.update_padded(&[]); + + let mut buf = [0_u8; 512]; + let mut read_length = 0; + while let Ok(read) = reader.read(&mut buf).await { + if read == 0 { + break; + } + mac.update_padded(&buf[..read]); + read_length += read; + } + debug!("{}", read_length); + + authenticate_lengths(&mut mac, 0, read_length); + + mac.verify(&tag).map_err(|_| Error::BinVerificationFailed)?; + } + + reader.seek(SeekFrom::Start(0)).await?; + + let body = AsyncReadBody::new(DecReader::new(reader, key, nonce, &[], tag)); let mut headers = HeaderMap::new(); headers.insert( @@ -481,15 +508,6 @@ async fn get_item( if let Some(content_type) = metadata.content_type.and_then(|x| x.parse().ok()) { headers.insert(header::CONTENT_TYPE, content_type); } - if let Some(etag) = metadata.etag.clone().and_then(|x| x.parse().ok()) { - headers.insert(header::ETAG, etag); - } - if let Some(digest) = metadata - .etag - .and_then(|x| format!("sha3-256={x}").parse().ok()) - { - headers.insert("Digest", digest); - } app_state .metrics @@ -559,3 +577,274 @@ pub struct IndexTemplate<'a> { #[derive(TemplateOnce)] #[template(path = "delete.stpl")] pub struct DeleteTemplate; + +pin_project! { + struct EncWriter { + #[pin] + inner: T, + buf: Vec, + to_mac: Vec, + cipher: ChaCha20, + mac: Poly1305, + tag: Option, + encrypted: usize, + authenticated_data_len: usize, + } +} + +impl EncWriter { + pub fn new(inner: T, key: Key, nonce: Nonce, authenticated_data: &[u8]) -> Self { + let mut cipher = ChaCha20::new(key.borrow(), nonce.borrow()); + let mut mac_key = poly1305::Key::default(); + cipher.apply_keystream(&mut mac_key); + + let mut mac = Poly1305::new(GenericArray::from_slice(&mac_key)); + mac_key.zeroize(); + + mac.update_padded(authenticated_data); + + // Set ChaCha20 counter to 1 + cipher.seek(64); + Self { + inner, + buf: vec![], + to_mac: vec![], + authenticated_data_len: authenticated_data.len(), + encrypted: 0, + cipher, + mac, + tag: None, + } + } + pub fn tag(&self) -> Option<&Tag> { + self.tag.as_ref() + } + pub fn size(&self) -> usize { + self.encrypted + } +} + +impl AsyncWrite for EncWriter { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let me = self.project(); + + if me.buf.len() > 16 { + return Poll::Pending; + } + + let mut buf = Vec::from(buf); + + me.cipher.apply_keystream(&mut buf); + + me.to_mac.extend_from_slice(&buf); + let macable = me.to_mac.drain(..align(buf.len(), 16)).collect::>(); + debug!("645: {}", macable.len() % 16); + me.mac.update_padded(&macable); + + *me.encrypted += buf.len(); + let buf_len = buf.len(); + + me.buf.append(&mut buf); + + if let Poll::Ready(Ok(written)) = me.inner.poll_write(cx, me.buf) { + me.buf.drain(..written); + } + + debug!("after write: {}", buf_len); + + Poll::Ready(Ok(buf_len)) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let me = self.project(); + + debug!("flush"); + + if !me.buf.is_empty() { + let written = ready!(me.inner.poll_write(cx, me.buf))?; + me.buf.drain(..written); + Poll::Pending + } else { + me.inner.poll_flush(cx) + } + } + + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let me = self.project(); + + debug!("shutdown"); + + if !me.buf.is_empty() { + let written = ready!(me.inner.poll_write(cx, me.buf))?; + me.buf.drain(..written); + Poll::Pending + } else if me.tag.is_none() { + me.mac.update_padded(me.to_mac); + me.to_mac.clear(); + + authenticate_lengths(me.mac, *me.authenticated_data_len, *me.encrypted); + + *me.tag = Some(me.mac.clone().finalize()); + Poll::Pending + } else { + me.inner.poll_shutdown(cx) + } + } +} + +pin_project! { + struct DecReader { + #[pin] + inner: T, + buf: Vec, + cipher: ChaCha20, + mac: Poly1305, + decrypted: usize, + authenticated_data_len: usize, + tag: Tag, + state: DecReaderState, + } +} + +impl DecReader { + pub fn new(inner: T, key: Key, nonce: Nonce, authenticated_data: &[u8], tag: Tag) -> Self { + let mut cipher = ChaCha20::new(key.borrow(), nonce.borrow()); + let mut mac_key = poly1305::Key::default(); + cipher.apply_keystream(&mut mac_key); + + let mut mac = Poly1305::new(GenericArray::from_slice(&mac_key)); + mac_key.zeroize(); + + mac.update_padded(authenticated_data); + + // Set ChaCha20 counter to 1 + cipher.seek(64); + Self { + inner, + buf: vec![0_u8; 512], + cipher, + mac, + authenticated_data_len: authenticated_data.len(), + decrypted: 0, + state: DecReaderState::Running, + tag, + } + } +} + +#[derive(Debug)] +enum DecReaderState { + Running, + Finished, + Failed, +} + +impl AsyncRead for DecReader { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let me = self.project(); + + //TODO move buf.filled to struct + + match me.state { + DecReaderState::Finished => Poll::Ready(Ok(())), + DecReaderState::Failed => Poll::Ready(Err(std::io::Error::new( + ErrorKind::InvalidData, + "decryption failed", + ))), + DecReaderState::Running => { + let changed = { + me.buf.extend_from_slice(&[0_u8; 512]); + let mut read_buf = ReadBuf::new(me.buf); + + let prev_buf_len = read_buf.filled().len(); + ready!(me.inner.poll_read(cx, &mut read_buf))?; + let buf_len = read_buf.filled().len(); + drop(read_buf); + me.buf.drain(buf_len..); + prev_buf_len == buf_len + }; + + if changed { + let decryptable = me.buf; + me.mac.update_padded(decryptable); + me.cipher.apply_keystream(decryptable); + + *me.decrypted += decryptable.len(); + + buf.put_slice(decryptable); + + authenticate_lengths(me.mac, *me.authenticated_data_len, *me.decrypted); + + match me.mac.clone().verify(me.tag) { + Ok(_) => { + *me.state = DecReaderState::Finished; + + warn!("finished"); + Poll::Ready(Ok(())) + } + Err(_) => { + *me.state = DecReaderState::Failed; + warn!("decryption failed"); + + Poll::Ready(Err(std::io::Error::new( + ErrorKind::InvalidData, + "decryption failed", + ))) + } + } + } else { + let (decryptable, residual) = { + let decryptable_buffer_len = align(min(buf.remaining(), me.buf.len()), 16); + me.buf.split_at_mut(decryptable_buffer_len) + }; + if !decryptable.is_empty() { + me.mac.update_padded(decryptable); + me.cipher.apply_keystream(decryptable); + + *me.decrypted += decryptable.len(); + + buf.put_slice(decryptable); + + { + let decryptable_len = decryptable.len(); + let residual_len = residual.len(); + me.buf.copy_within(decryptable_len.., 0); + me.buf.drain(residual_len..); + } + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } + } + } + } +} + +fn min(a: usize, b: usize) -> usize { + match a < b { + true => a, + false => b, + } +} +fn align(val: usize, m: usize) -> usize { + val - (val % m) +} + +fn authenticate_lengths(mac: &mut Poly1305, associated_data_len: usize, buffer_len: usize) { + let mut block = GenericArray::default(); + block[..8].copy_from_slice(&associated_data_len.to_le_bytes()); + block[8..].copy_from_slice(&buffer_len.to_le_bytes()); + mac.update(&[block]); +} diff --git a/server/src/metadata.rs b/server/src/metadata.rs index 2df9f70..228714a 100644 --- a/server/src/metadata.rs +++ b/server/src/metadata.rs @@ -9,7 +9,7 @@ use crate::Error; pub struct Metadata { pub subject: String, pub nonce: String, - pub etag: Option, + pub tag: Option, pub size: Option, pub content_type: Option, pub expires_at: u64, // seconds since UNIX_EPOCH diff --git a/server/src/web_util.rs b/server/src/web_util.rs index 1ecc505..fc9ba39 100644 --- a/server/src/web_util.rs +++ b/server/src/web_util.rs @@ -29,8 +29,6 @@ pin_project! { capacity: usize, // chacha20 cipher cipher: ChaCha20, - // hasher to verify file integrity - hasher: Sha3_256, // hash to verify against target_hash: String, // id of the file for logging purposes @@ -49,8 +47,7 @@ impl DecryptingStream { buf: BytesMut::new(), capacity: 1 << 22, // 4 MiB cipher, - hasher: Sha3_256::new(), - target_hash: metadata.etag.clone().unwrap_or_default(), + target_hash: metadata.tag.clone().unwrap_or_default(), id, size: metadata.size.unwrap_or_default(), progress: 0, @@ -94,7 +91,6 @@ impl Stream for DecryptingStream { // decrypt the chunk using chacha this.cipher.apply_keystream(&mut chunk); // update the sha3 hasher - this.hasher.update(&chunk); // track progress *this.progress += n as u64; if self.progress_check() == DecryptingStreamProgress::Failed {