From 0f5f4e7f8cd3133ce367bc3a5993c5549b2d5bf1 Mon Sep 17 00:00:00 2001 From: Paul Z Date: Mon, 5 Jun 2023 22:53:55 +0200 Subject: [PATCH] s3 storage backend replacement --- Cargo.lock | 503 +++++++++++++++++++++++++++++++++------- Cargo.toml | 7 +- src/item_explanation.md | 2 +- src/main.rs | 406 ++++++++++++++++++-------------- src/metadata.rs | 59 ----- 5 files changed, 654 insertions(+), 323 deletions(-) delete mode 100644 src/metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 82d57ac..36393bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.0.1" @@ -72,12 +83,54 @@ dependencies = [ "syn 2.0.18", ] +[[package]] +name = "attohttpc" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +dependencies = [ + "http", + "log", + "rustls 0.20.8", + "serde", + "serde_json", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-creds" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" +dependencies = [ + "attohttpc", + "dirs", + "log", + "quick-xml", + "rust-ini", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba" +dependencies = [ + "thiserror", +] + [[package]] name = "axum" version = "0.6.18" @@ -209,13 +262,14 @@ version = "0.1.0" dependencies = [ "axum", "axum_oidc", - "chrono", "dotenvy", "futures-util", + "log", "markdown", - "parse_duration", + "pretty_env_logger", "rand", "render", + "rust-s3", "serde", "serde_cbor", "thiserror", @@ -276,11 +330,8 @@ checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ "android-tzdata", "iana-time-zone", - "js-sys", "num-traits", "serde", - "time 0.1.45", - "wasm-bindgen", "winapi", ] @@ -311,7 +362,7 @@ dependencies = [ "percent-encoding", "rand", "subtle", - "time 0.3.21", + "time", "version_check", ] @@ -420,6 +471,32 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "dotenvy" version = "0.15.7" @@ -481,6 +558,40 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "ff" version = "0.12.1" @@ -506,6 +617,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -513,6 +639,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -521,6 +648,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -556,9 +694,11 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -585,7 +725,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -640,6 +780,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] [[package]] name = "headers" @@ -681,6 +824,18 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hkdf" version = "0.12.3" @@ -739,6 +894,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.26" @@ -771,7 +932,7 @@ checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", - "rustls", + "rustls 0.21.1", "tokio", "tokio-rustls", ] @@ -834,12 +995,35 @@ dependencies = [ "generic-array", ] +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "ipnet" version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -885,6 +1069,12 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "lock_api" version = "0.4.9" @@ -918,6 +1108,23 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -930,6 +1137,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minidom" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278" +dependencies = [ + "rxml", +] + [[package]] name = "mio" version = "0.8.8" @@ -937,35 +1153,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.48.0", ] -[[package]] -name = "num" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-bigint-dig" version = "0.8.2" @@ -983,16 +1174,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "num-complex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-integer" version = "0.1.45" @@ -1014,18 +1195,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-rational" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" -dependencies = [ - "autocfg", - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.15" @@ -1042,7 +1211,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -1118,6 +1287,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown", +] + [[package]] name = "p256" version = "0.11.1" @@ -1163,17 +1342,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "parse_duration" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d" -dependencies = [ - "lazy_static", - "num", - "regex", -] - [[package]] name = "pem-rfc7468" version = "0.6.0" @@ -1267,6 +1435,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_env_logger" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" +dependencies = [ + "env_logger", + "log", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1300,6 +1478,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.28" @@ -1348,6 +1536,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.8.3" @@ -1407,17 +1606,19 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.1", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-rustls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots", "winreg", @@ -1470,6 +1671,75 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.1", + "bytes", + "cfg-if", + "futures", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "minidom", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_derive", + "sha2", + "thiserror", + "time", + "tokio", + "tokio-stream", + "url", +] + +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "rustls" version = "0.21.1" @@ -1507,6 +1777,23 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +[[package]] +name = "rxml" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7" +dependencies = [ + "bytes", + "rxml_validation", + "smartstring", +] + +[[package]] +name = "rxml_validation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530" + [[package]] name = "ryu" version = "1.0.13" @@ -1702,6 +1989,17 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "socket2" version = "0.4.9" @@ -1728,6 +2026,12 @@ dependencies = [ "der", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" @@ -1768,6 +2072,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.40" @@ -1788,17 +2101,6 @@ dependencies = [ "syn 2.0.18", ] -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "time" version = "0.3.21" @@ -1877,7 +2179,18 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" dependencies = [ - "rustls", + "rustls 0.21.1", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", "tokio", ] @@ -2039,12 +2352,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2117,6 +2424,19 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.63" @@ -2162,6 +2482,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 33e13b3..bd24432 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] tokio = { version = "1.27.0", features = ["full"] } -tokio-util = { version="0.7", features=["io"]} +tokio-util = { version="0.7"} futures-util = "0.3" axum = {version="0.6", features=["macros", "headers"]} serde = "1.0" @@ -17,6 +17,7 @@ thiserror = "1.0.40" rand = "0.8.5" dotenvy = "0.15" markdown = "0.3.0" -chrono = { version="0.4", features=["serde"]} -parse_duration = "2.1" axum_oidc = {git="https://git.zettoit.eu/pfz4/axum_oidc"} +rust-s3 = { version="0.33.0", features=["tokio-rustls-tls", "tags"], default_features=false } +log = "0.4.18" +pretty_env_logger = "0.5.0" diff --git a/src/item_explanation.md b/src/item_explanation.md index 9f54269..52043a2 100644 --- a/src/item_explanation.md +++ b/src/item_explanation.md @@ -4,7 +4,7 @@ All following requests can only read the uploaded data. To use the build-in link-shortener functionality you have to POST or PUT the URL with Content-Type: text/x-uri or Content-Type: text/uri-list. -To change the default expiration date of one month, you can use the `?ttl=` parameter. It supports durations like `60s`, `60m`, `24h`, `30d`, `12M`, `1y` or iso dates like `1970-01-01T00:00:00` in UTC timezone. +To change the default expiration date, you can use the `?ttl=` parameter. The following expiry classes are defined: . ## Upload a link `$ curl -H "Content-Type: text/x-uri" --data "https://example.com" ` diff --git a/src/main.rs b/src/main.rs index e2840ea..c3e4bef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,14 @@ -use std::{collections::BTreeMap, env, str::FromStr, sync::Arc, time::Duration}; +use std::{ + collections::HashSet, + env, + io::ErrorKind, + task::{Context, Poll}, + time::Duration, +}; use axum::{ - body::StreamBody, + body::{Bytes, StreamBody}, + debug_handler, extract::{BodyStream, FromRef, Path, Query, State}, headers::ContentType, http::{header, HeaderMap, StatusCode}, @@ -10,36 +17,32 @@ use axum::{ Router, TypedHeader, }; use axum_oidc::{ClaimsExtractor, EmptyAdditionalClaims, Key, OidcApplication}; -use chrono::{NaiveDateTime, Utc}; -use futures_util::StreamExt; -use metadata::Metadata; +use futures_util::{Stream, StreamExt, TryStreamExt}; +use log::{info, warn}; +use rand::{distributions::Alphanumeric, Rng}; use render::{html, raw}; +use s3::{creds::Credentials, error::S3Error, request::ResponseDataStream, Bucket}; use serde::Deserialize; -use tokio::{ - fs::{self, File}, - io::{AsyncReadExt, AsyncWriteExt}, - sync::Mutex, -}; -use tokio_util::io::ReaderStream; - -pub mod metadata; +use tokio_util::io::StreamReader; // RFC 7230 section 3.1.1 // It is RECOMMENDED that all HTTP senders and recipients // support, at a minimum, request-line lengths of 8000 octets. -const HTTP_URL_MAXLENGTH: u64 = 8000; +const HTTP_URL_MAXLENGTH: i64 = 8000; // support the RFC2483 with text/uri-list and the inofficial text/x-uri mimetype const HTTP_URL_MIMETYPES: [&str; 2] = ["text/x-uri", "text/uri-list"]; #[derive(Debug, thiserror::Error)] enum Error { + #[error("s3 error: {:?}", 1)] + S3(#[from] S3Error), + #[error("url is invalid utf8")] + UrlUtf8Invalid, #[error("item could not be found")] ItemNotFound, - #[error("errorfile exists")] + #[error("file exists")] DataFileExists, - #[error("datafile without metafile")] - DataFileWithoutMetaFile, #[error("could not parse ttl")] ParseTtl, @@ -49,13 +52,14 @@ type HandlerResult = Result; impl IntoResponse for Error { fn into_response(self) -> axum::response::Response { match self { - Self::ItemNotFound => (StatusCode::NOT_FOUND, "item could not be found"), - Self::DataFileExists => (StatusCode::CONFLICT, "item already has data"), - Self::DataFileWithoutMetaFile => { - println!("WARN: a data file without meta file exists."); - (StatusCode::INTERNAL_SERVER_ERROR, "internal server error") + Self::S3(e) => { + warn!("S3 Error: {:?}", e); + (StatusCode::INTERNAL_SERVER_ERROR, "s3 error") } - Self::ParseTtl => (StatusCode::BAD_REQUEST, "could not parse ttl"), + Self::UrlUtf8Invalid => (StatusCode::INTERNAL_SERVER_ERROR, "url is not valid utf8"), + Self::ItemNotFound => (StatusCode::NOT_FOUND, "bin could not be found"), + Self::DataFileExists => (StatusCode::CONFLICT, "bin already has data"), + Self::ParseTtl => (StatusCode::BAD_REQUEST, "invalid ttl class"), } .into_response() } @@ -63,10 +67,11 @@ impl IntoResponse for Error { #[derive(Clone)] pub struct AppState { - path: String, application_base: String, oidc_application: OidcApplication, - expire: Arc>>, + bucket: Bucket, + lifecycle_classes: HashSet, + default_lifecycle_class: String, } impl FromRef for OidcApplication { @@ -78,6 +83,7 @@ impl FromRef for OidcApplication { #[tokio::main] async fn main() { dotenvy::dotenv().ok(); + pretty_env_logger::init(); let application_base = env::var("APPLICATION_BASE").expect("APPLICATION_BASE env var"); let issuer = env::var("ISSUER").expect("ISSUER env var"); @@ -86,16 +92,9 @@ async fn main() { let scopes = env::var("SCOPES") .expect("SCOPES env var") .split(' ') - .into_iter() .map(|x| x.to_owned()) .collect::>(); - let expire = Arc::new(Mutex::new(BTreeMap::new())); - { - let mut expire = expire.lock().await; - load_expire("data", &mut expire).await; - } - let oidc_application = OidcApplication::::create( application_base.parse().unwrap(), issuer.to_string(), @@ -107,15 +106,44 @@ async fn main() { .await .unwrap(); + let bucket = Bucket::new( + &env::var("S3_BUCKET").expect("S3_BUCKET env var"), + env::var("S3_REGION") + .expect("S3_REGION env var") + .parse() + .unwrap(), + Credentials::new( + Some(&env::var("S3_ACCESS_KEY").expect("S3_ACCESS_KEY env var")), + Some(&env::var("S3_SECRET_KEY").expect("S3_SECRET_KEY env var")), + None, + None, + None, + ) + .unwrap(), + ) + .unwrap() + .with_path_style() + .with_request_timeout(Duration::from_secs(60 * 60 * 6)); + + let lifecycle_classes = std::env::var("LIFECYCLE_CLASSES") + .expect("LIFECYCLE_CLASSES env var") + .split(',') + .map(|x| x.to_string()) + .collect::>(); + let default_lifecycle_class = + std::env::var("DEFAULT_LIFECYCLE_CLASS").expect("DEFAULT_LIFECYCLE_CLASS env var"); + if !lifecycle_classes.contains(&default_lifecycle_class) { + panic!("DEFAULT_LIFECYCLE_CLASS must be an element of LIFECYCLE_CLASSES"); + } + let state: AppState = AppState { - path: "data".to_string(), application_base, oidc_application, - expire: expire.clone(), + bucket, + lifecycle_classes, + default_lifecycle_class, }; - tokio::spawn(async move { expire_thread("data".to_string(), expire).await }); - let app = Router::new() .route("/", get(get_index)) .route("/:id", get(get_item).post(post_item).put(post_item)) @@ -129,12 +157,34 @@ async fn main() { async fn get_index( State(app_state): State, ClaimsExtractor(claims): ClaimsExtractor, -) -> impl IntoResponse { +) -> Result { let subject = claims.subject().to_string(); - let (id, _) = Metadata::create(&app_state.path, subject).await.unwrap(); + //generate id + let id = rand::thread_rng() + .sample_iter(Alphanumeric) + .take(8) + .map(char::from) + .collect::(); - Redirect::temporary(&format!("{}{}", app_state.application_base, id)) + app_state.bucket.put_object(&id, &[]).await?; + app_state + .bucket + .put_object_tagging( + &id, + &[ + ("ttl".to_string(), app_state.default_lifecycle_class.clone()), + ("subject".to_string(), subject), + ], + ) + .await?; + + info!("created bin {id}"); + + Ok(Redirect::temporary(&format!( + "{}{}", + app_state.application_base, id + ))) } #[derive(Deserialize)] @@ -147,126 +197,161 @@ async fn post_item( Query(params): Query, State(app_state): State, content_type: Option>, - mut stream: BodyStream, + stream: BodyStream, ) -> HandlerResult { let id = sanitize_id(id); - let mut metadata = Metadata::from_file(&app_state.path, &id) - .await - .map_err(|_| Error::ItemNotFound)?; + let metadata = app_state.bucket.head_object(&id).await?.0; - if fs::metadata(&format!("{}/{}.data", app_state.path, &id)) - .await - .is_err() - { - let expires_at = if let Some(ttl) = params.ttl { - if let Ok(duration) = parse_duration::parse(&ttl) { - Ok(Utc::now().naive_utc() + chrono::Duration::from_std(duration).unwrap()) - } else if let Ok(ttl) = chrono::NaiveDateTime::from_str(&ttl) { - Ok(ttl) - } else { - Err(Error::ParseTtl) - } - } else { - Ok(Utc::now().naive_utc() + chrono::Duration::days(30)) - }?; - - let mut data_file = File::create(&format!("{}/{}.data", app_state.path, &id)) - .await - .unwrap(); - - while let Some(chunk) = stream.next().await { - let buf = chunk.map(|x| x.to_vec()).unwrap_or_default(); - data_file.write_all(&buf).await.unwrap(); - } - - metadata.mimetype = content_type.map(|x| x.0.to_string()); - metadata.ttl = Some(expires_at); - metadata.to_file(&app_state.path, &id).await.unwrap(); - - app_state.expire.lock().await.insert(expires_at, id); - - Ok((StatusCode::CREATED, "OK")) - } else { - Err(Error::DataFileExists) + if metadata.e_tag.is_none() { + return Err(Error::ItemNotFound); } + if let Some(content_length) = metadata.content_length { + if content_length > 0 { + return Err(Error::DataFileExists); + } + } + + let ttl = if let Some(ttl) = ¶ms.ttl { + if !app_state.lifecycle_classes.contains(ttl) { + return Err(Error::ParseTtl); + } + ttl + } else { + &app_state.default_lifecycle_class + }; + + let tags = app_state.bucket.get_object_tagging(&id).await?.0; + + let mut reader = + StreamReader::new(stream.map_err(|e| std::io::Error::new(ErrorKind::Other, e.to_string()))); + let status_code = match content_type { + Some(content_type) => { + app_state + .bucket + .put_object_stream_with_content_type(&mut reader, &id, &content_type.to_string()) + .await + } + None => app_state.bucket.put_object_stream(&mut reader, &id).await, + }?; + + let status_code = StatusCode::from_u16(status_code).unwrap(); + + let subject = tags + .iter() + .find(|x| x.key() == "subject") + .map(|x| x.value()) + .unwrap_or_default(); + + app_state + .bucket + .put_object_tagging( + &id, + &[ + ("ttl".to_string(), ttl.to_string()), + ("subject".to_string(), subject), + ], + ) + .await?; + + info!("bin {id} is now read only"); + + Ok(( + status_code, + format!( + "{}\n", + status_code + .canonical_reason() + .unwrap_or(&status_code.to_string()) + ), + )) } + +#[debug_handler] async fn get_item( Path(id): Path, State(app_state): State, ) -> HandlerResult { let id = sanitize_id(id); - let metadata = Metadata::from_file(&app_state.path, &id).await.ok(); + let metadata = app_state.bucket.head_object(&id).await?.0; - let data_file = File::open(&format!("{}/{}.data", app_state.path, &id)) - .await - .ok(); - - match (metadata, data_file) { - (None, None) => Err(Error::ItemNotFound), - (Some(_), None) => { - let body = include_str!("item_explanation.md").replace( + if metadata.e_tag.is_none() { + return Err(Error::ItemNotFound); + } + if metadata.content_length.is_none() || metadata.content_length == Some(0) { + let body = include_str!("item_explanation.md") + .replace( "", &format!("{}{}", app_state.application_base, id), + ) + .replace( + "", + &app_state + .lifecycle_classes + .iter() + .map(|x| x.to_string()) + .reduce(|acc, e| acc + ", " + e.as_str()) + .unwrap_or_default(), ); - let body = markdown::to_html(&body); - let body = html! { - - - {"zettoit bin"} - - - -
- {raw!(body.as_str())} -
- - - }; - Ok((StatusCode::ACCEPTED, Html(body)).into_response()) - } - (Some(metadata), Some(mut data_file)) => { - let data_file_metadata = fs::metadata(&format!("{}/{}.data", app_state.path, &id)) - .await - .unwrap(); - if HTTP_URL_MIMETYPES.contains(&metadata.mimetype.as_deref().unwrap_or("")) - && data_file_metadata.len() <= HTTP_URL_MAXLENGTH - { - let mut url = String::new(); - data_file.read_to_string(&mut url).await.unwrap(); + let body = markdown::to_html(&body); + let body = html! { + + + {"zettoit bin"} + + + +
+ {raw!(body.as_str())} +
+ + + }; + Ok((StatusCode::ACCEPTED, Html(body)).into_response()) + } else if let Some(content_length) = metadata.content_length { + if HTTP_URL_MIMETYPES.contains(&metadata.content_type.as_deref().unwrap_or("")) + && content_length <= HTTP_URL_MAXLENGTH + { + let file = app_state.bucket.get_object(&id).await?; + let url = String::from_utf8(file.to_vec()).map_err(|_| Error::UrlUtf8Invalid)?; - // Use the first line that doesn't start with a # to be compliant with RFC2483. - let url = url - .lines() - .into_iter() - .find(|x| !x.starts_with('#')) - .unwrap_or(""); + // Use the first line that doesn't start with a # to be compliant with RFC2483. + let url = url.lines().find(|x| !x.starts_with('#')).unwrap_or(""); - Ok(Redirect::temporary(url).into_response()) - } else { - let reader = ReaderStream::new(data_file); - let body = StreamBody::new(reader); + Ok(Redirect::temporary(url).into_response()) + } else { + let file_stream = app_state.bucket.get_object_stream(&id).await.unwrap(); - let mut headers = HeaderMap::new(); - headers.insert( - header::CONTENT_TYPE, - metadata - .mimetype - .as_deref() - .unwrap_or("text/plain") - .parse() - .unwrap(), - ); - headers.insert( - header::CONTENT_LENGTH, - data_file_metadata.len().to_string().parse().unwrap(), - ); + let body = StreamBody::new(ResponseStream(std::sync::Mutex::new(file_stream))); - Ok((StatusCode::OK, headers, body).into_response()) + let mut headers = HeaderMap::new(); + headers.insert( + header::CONTENT_LENGTH, + metadata.content_length.unwrap().into(), + ); + + if let Some(content_type) = metadata.content_type { + headers.insert(header::CONTENT_TYPE, content_type.parse().unwrap()); } + if let Some(etag) = metadata.e_tag { + headers.insert(header::ETAG, etag.parse().unwrap()); + } + if let Some(content_encoding) = metadata.content_encoding { + headers.insert(header::CONTENT_ENCODING, content_encoding.parse().unwrap()); + } + if let Some(content_language) = metadata.content_language { + headers.insert(header::CONTENT_LANGUAGE, content_language.parse().unwrap()); + } + if let Some(cache_control) = metadata.cache_control { + headers.insert(header::CACHE_CONTROL, cache_control.parse().unwrap()); + } + + Ok((StatusCode::OK, headers, body).into_response()) } - (None, Some(_)) => Err(Error::DataFileWithoutMetaFile), + } else { + // logically should not happen + panic!("logic contradiction"); } } @@ -277,42 +362,17 @@ fn sanitize_id(id: String) -> String { .collect() } -async fn expire_thread(path: String, expire_dates: Arc>>) { - loop { - { - let now = Utc::now().naive_utc(); - let mut expire_dates = expire_dates.lock().await; - let mut to_delete = vec![]; - for (expire, id) in expire_dates.iter() { - if expire < &now { - println!("removing {id}"); - fs::remove_file(&format!("{}/{}.meta", &path, &id)) - .await - .unwrap(); - let _ = fs::remove_file(&format!("{}/{}.data", &path, &id)).await; +struct ResponseStream(std::sync::Mutex); +impl Stream for ResponseStream { + type Item = Result; - to_delete.push(*expire); - } else { - break; - } - } - for expire in to_delete.iter() { - expire_dates.remove(expire); - } - } - tokio::time::sleep(Duration::from_secs(1)).await; - } -} -async fn load_expire(path: &str, expire: &mut BTreeMap) { - let mut files = fs::read_dir(path).await.unwrap(); - while let Some(file) = files.next_entry().await.unwrap() { - let name = file.file_name().into_string().unwrap_or(String::new()); - if name.ends_with(".meta") { - let id = name.replace(".meta", ""); - let metadata = Metadata::from_file(path, &id).await.unwrap(); - if let Some(ttl) = metadata.ttl { - expire.insert(ttl, id); - } - } + fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.0 + .lock() + .unwrap() + .bytes() + .poll_next_unpin(cx) + .map(|x| x.map(Ok)) } } +unsafe impl Send for ResponseStream {} diff --git a/src/metadata.rs b/src/metadata.rs deleted file mode 100644 index af8e58d..0000000 --- a/src/metadata.rs +++ /dev/null @@ -1,59 +0,0 @@ -use chrono::{NaiveDateTime, Utc}; -use rand::{distributions::Alphanumeric, Rng}; -use serde::{Deserialize, Serialize}; -use tokio::{ - fs::File, - io::{AsyncReadExt, AsyncWriteExt}, -}; - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error("io error {:?}", 0)] - Io(#[from] tokio::io::Error), - - #[error("cbor error: {:?}", 0)] - Cbor(#[from] serde_cbor::Error), -} - -#[derive(Serialize, Deserialize)] -pub struct Metadata { - pub subject: String, - pub ttl: Option, - pub mimetype: Option, - pub sha256: Option, - pub sha512: Option, -} - -impl Metadata { - pub async fn create(path: &str, subject: String) -> Result<(String, Self), Error> { - let id = rand::thread_rng() - .sample_iter(Alphanumeric) - .take(8) - .map(char::from) - .collect::(); - - let metadata = Metadata { - subject, - mimetype: None, - ttl: Some(Utc::now().naive_utc() + chrono::Duration::days(1)), - sha256: None, - sha512: None, - }; - - metadata.to_file(path, &id).await.unwrap(); - - Ok((id, metadata)) - } - pub async fn from_file(path: &str, id: &str) -> Result { - let mut metadata_file = File::open(&format!("{}/{}.meta", path, id)).await?; - let mut metadata = Vec::new(); - metadata_file.read_to_end(&mut metadata).await.unwrap(); - Ok(serde_cbor::from_slice(&metadata)?) - } - pub async fn to_file(&self, path: &str, id: &str) -> Result<(), Error> { - let metadata = serde_cbor::to_vec(self).unwrap(); - let mut metadata_file = File::create(&format!("{}/{}.meta", path, id)).await?; - metadata_file.write_all(&metadata).await?; - Ok(()) - } -}