diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 49648e7..d69832b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -31,15 +31,17 @@ jobs: target/ key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }} - # - name: Install WASM toolchain - # run: rustup target add wasm32-unknown-unknown + - name: Install WASM toolchain + run: rustup target add wasm32-unknown-unknown - name: Check project run: cargo clippy --workspace --all-targets --all-features -- -D warnings - # - name: Check project (WASM) - # # Note lack of --all-targets here because tests and examples are not wasm compatible - # run: cargo clippy --workspace --all-features --target wasm32-unknown-unknown -- -D warnings + - name: Check project (WASM) + # Note lack of --all-targets here because tests and examples are not wasm compatible + # (the CLI binary compiles as an empty stub). getrandom 0.3 requires selecting its + # JS backend explicitly on wasm32-unknown-unknown. + run: RUSTFLAGS='--cfg getrandom_backend="wasm_js"' cargo clippy --workspace --all-features --target wasm32-unknown-unknown -- -D warnings test: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 6e002b4..b0ffc72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -843,8 +843,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -854,8 +856,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -891,6 +895,8 @@ dependencies = [ "cssparser 0.33.0", "epub-builder", "fontdb", + "getrandom 0.2.15", + "getrandom 0.3.1", "glob", "html-escape", "html5ever", @@ -3183,6 +3189,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" dependencies = [ "getrandom 0.3.1", + "js-sys", + "wasm-bindgen", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index dc51f98..215e965 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2024" [dependencies] chrono = { version = "0.4", features = ["serde"] } -glob = "0.3" log = "0.4" mime = "0.3" rand = "0.8" @@ -18,10 +17,8 @@ regex = "1" sha2 = "0.10" uuid = { version = "1", features = ["v4"] } -reqwest = { version = "0.12", features = ["json"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -tokio = { version = "1", features = ["full"] } fontdb = "0.16" resvg = "0.41" @@ -43,7 +40,22 @@ xml5ever = "0.18" image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg", "png", "webp"] } +slug = "0.1" + +# Native-only dependencies: networking, async runtime, filesystem cache, and the CLI. +# The library builds for wasm32 without them (see src/lib.rs cfg gates); the +# `Thread::to_epub_remote_images` path is fully available on wasm. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +glob = "0.3" +reqwest = { version = "0.12", features = ["json"] } +tokio = { version = "1", features = ["full"] } clap = { version = "4", features = ["derive"] } simple_logger = "4" -slug = "0.1" rpassword = "7.3.1" + +# On wasm32-unknown-unknown, getrandom needs its JS backends enabled +# (pulled in transitively via `rand` and `uuid`). +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom02 = { package = "getrandom", version = "0.2", features = ["js"] } +getrandom03 = { package = "getrandom", version = "0.3", features = ["wasm_js"] } +uuid = { version = "1", features = ["js"] } diff --git a/src/cached.rs b/src/cached.rs index 1736171..3cf4fdc 100644 --- a/src/cached.rs +++ b/src/cached.rs @@ -57,14 +57,13 @@ impl Replies { ) -> Result, Vec>, Box> { let cache_path = Self::cache_key(id); - if !invalidate_cache { - if let Ok(data) = std::fs::read(&cache_path) { - let parsed: Result> = - serde_json::from_slice(&data).unwrap(); + if !invalidate_cache + && let Ok(data) = std::fs::read(&cache_path) + { + let parsed: Result> = serde_json::from_slice(&data).unwrap(); - if let Ok(replies) = parsed { - return Ok(Ok(replies.0)); - } + if let Ok(replies) = parsed { + return Ok(Ok(replies.0)); } } @@ -88,14 +87,14 @@ impl BoardPosts { ) -> Result, Vec>, Box> { let cache_path = Self::cache_key(id); - if !invalidate_cache { - if let Ok(data) = std::fs::read(&cache_path) { - let parsed: Result, Vec> = - serde_json::from_slice(&data).unwrap(); + if !invalidate_cache + && let Ok(data) = std::fs::read(&cache_path) + { + let parsed: Result, Vec> = + serde_json::from_slice(&data).unwrap(); - if let Ok(posts) = parsed { - return Ok(Ok(posts)); - } + if let Ok(posts) = parsed { + return Ok(Ok(posts)); } } @@ -124,10 +123,10 @@ impl Icon { return Err("No url provided for this icon".into()); }; - if !invalidate_cache { - if let Ok((mime, data)) = read_image_file(Self::cache_key(*id, "*")) { - return Ok((mime, data)); - } + if !invalidate_cache + && let Ok((mime, data)) = read_image_file(Self::cache_key(*id, "*")) + { + return Ok((mime, data)); } log::info!("Downloading icon {id} from {url}"); @@ -232,10 +231,10 @@ pub async fn download_cached_image( let hash = url_hash(url); - if !invalidate_cache { - if let Ok((mime, data)) = read_image_file(image_cache_key(&hash, "*")) { - return Ok((mime, data)); - } + if !invalidate_cache + && let Ok((mime, data)) = read_image_file(image_cache_key(&hash, "*")) + { + return Ok((mime, data)); } log::info!("Downloading image {hash} from {url}"); @@ -261,12 +260,12 @@ async fn get_cached_glowfic( where T: DeserializeOwned + Serialize, { - if !invalidate_cache { - if let Ok(data) = std::fs::read(cache_path) { - let parsed: Result> = serde_json::from_slice(&data).unwrap(); - if parsed.is_ok() { - return Ok(parsed); - } + if !invalidate_cache + && let Ok(data) = std::fs::read(cache_path) + { + let parsed: Result> = serde_json::from_slice(&data).unwrap(); + if parsed.is_ok() { + return Ok(parsed); } } let response = crate::api::get_glowfic(url).await?; diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..9d628ff --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,375 @@ +use clap::Parser; +use std::{ + fmt::Display, + path::{Path, PathBuf}, +}; + +use glowpub::{ + Board, Thread, + api::BoardPosts, + cached::write_if_changed, + generate::Options, + types::{Continuity, Section}, +}; + +const DEFAULT_OUTPUT_DIR: &str = "./books"; + +/// Download and process Glowfic posts into epub and html files. +#[derive(Debug, Parser)] +enum Command { + /// Download and process a single post. + Post { + /// The id of the Glowfic post. + /// Can be found in the URL: https://glowfic.com/posts/ + post_id: u64, + + #[command(flatten)] + options: CliOptions, + }, + /// Download and process an entire board. + Board { + /// The id of the Glowfic board. + /// Can be found in the URL: https://glowfic.com/boards/ + board_id: u64, + + #[command(flatten)] + options: CliOptions, + + /// If enabled, the board will be processed into a single epub file instead of being split by post. + #[clap(long)] + single_file: bool, + }, +} +impl Command { + fn options(&self) -> CliOptions { + match self { + Command::Post { options, .. } | Command::Board { options, .. } => options.clone(), + } + } +} + +#[derive(Debug, Clone, Parser)] +struct CliOptions { + /// Reuse already downloaded data. Images are always cached. + #[clap(long)] + use_cache: bool, + + /// Simplify character and user names to improve text-to-speech output. + #[clap(long)] + text_to_speech: bool, + + ///
tags can be hard to use on e-readers, this option forces them to always seem open. + /// + /// (Under the hood, it replaces the
tag with a
, and with

, + /// it also preprends `▼ ` to the

tag to make it similar to an open
tag.) + #[clap(long)] + flatten_details: Option, + + /// When inlining the images into the epub file, this will convert all images into jpeg files. + /// In general this will result in considerably smaller files if the images are not already jpegs. + /// (Does not affect SVGs.) + #[clap(long)] + jpeg: bool, + + /// When inlining icons into the epub file, this will scale all icon images above the provided width down to that width. + /// Defaults to "100" if no value is provided. + /// (Does not affect SVGs or non-icon images.) + #[clap(long)] + resize_icons: Option>, + + /// Output files in this directory (e.g. `--output-dir=~/glowfic`). + /// Note that this can flood the directory if used with `board` but without `--single-file`. + /// Files will be placed in format-specific subdirectories if this option is not set, or if `--output-format` is `both` (the default). + #[clap(long)] + output_dir: Option, + + /// Specify the output directory layout. + #[clap(long, default_value_t = OutputDirLayout::default())] + output_dir_layout: OutputDirLayout, + + /// Determines which file-types are created by the program. + #[clap(long, default_value_t = OutputFormat::default())] + output_format: OutputFormat, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +enum FlattenDetails { + /// The default option. No
tags will be flattened. + #[default] + None, + /// All
tags will be flattened. + All, + /// Only
tags in epubs will be flattened. + Mixed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +enum OutputDirLayout { + /// The default option. Output files will be placed in a nested subdirectory based on their board. + #[default] + Nested, + /// Output files will be placed directly in the output directory, and will have their board information included in the filename. + Flat, +} +impl Display for OutputDirLayout { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Nested => write!(f, "nested"), + Self::Flat => write!(f, "flat"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +enum OutputFormat { + /// The default option. Both HTML and Epub files will be created. + /// Output files will be placed in "html" and "epub" subdirectories, respectively. + #[default] + Both, + /// Only Epub files will be created. + Epub, + /// Only HTML files will be created. + Html, + /// No output files will be created. + None, +} +impl OutputFormat { + fn epub(self) -> bool { + matches!(self, Self::Epub | Self::Both) + } + fn html(self) -> bool { + matches!(self, Self::Html | Self::Both) + } +} +impl Display for OutputFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Both => write!(f, "both"), + Self::Epub => write!(f, "epub"), + Self::Html => write!(f, "html"), + Self::None => write!(f, "none"), + } + } +} + +pub async fn main() { + simple_logger::init_with_level(log::Level::Info).unwrap(); + + let command = Command::parse(); + + let CliOptions { + use_cache, + text_to_speech, + flatten_details, + jpeg, + resize_icons, + output_dir, + output_dir_layout, + output_format, + } = command.options(); + + let resize_icons = resize_icons.map(|r| r.unwrap_or(100)); + + let mut html_output_dir = output_dir + .clone() + .unwrap_or_else(|| PathBuf::from(DEFAULT_OUTPUT_DIR)); + let mut epub_output_dir = output_dir + .clone() + .unwrap_or_else(|| PathBuf::from(DEFAULT_OUTPUT_DIR)); + + if output_format == OutputFormat::Both || output_dir.is_none() { + html_output_dir = html_output_dir.join("html/"); + epub_output_dir = epub_output_dir.join("epub/"); + } + + let epub_options = Options { + text_to_speech, + flatten_details: match flatten_details.unwrap_or_default() { + FlattenDetails::All | FlattenDetails::Mixed => true, + FlattenDetails::None => false, + }, + jpeg, + resize_icons, + }; + let html_options = Options { + text_to_speech, + flatten_details: match flatten_details.unwrap_or_default() { + FlattenDetails::All => true, + FlattenDetails::None | FlattenDetails::Mixed => false, + }, + jpeg, + resize_icons, + }; + + match command { + Command::Post { post_id, .. } => { + log::info!("Downloading post {post_id}"); + let thread = Thread::get_cached(post_id, !use_cache) + .await + .unwrap() + .unwrap(); + log::info!("Downloaded post {post_id} - {}", &thread.post.subject); + + log::info!("Caching all the icons..."); + thread.cache_all_icons(false).await; + + let name = { + let board = Board::get_cached(thread.post.board.id, !use_cache) + .await + .unwrap() + .unwrap(); + + let board_posts = BoardPosts::get_all_cached(board.id, !use_cache) + .await + .unwrap() + .unwrap(); + + thread_filename( + &thread, + &board, + board_posts.iter().map(|p| p.section.clone()), + OutputDirLayout::Flat, + ) + }; + + if output_format.html() { + log::info!("Generating html document {name}..."); + let path = html_output_dir.join(format!("{name}.html")); + write(path, thread.to_single_html_page(html_options)); + } + + if output_format.epub() { + log::info!("Generating epub document {name}..."); + let path = epub_output_dir.join(format!("{name}.epub")); + write(path, thread.to_epub(epub_options).await.unwrap()); + } + } + Command::Board { + board_id, + single_file: false, + .. + } => { + log::info!("Downloading board/continuity {board_id}..."); + let continuity = Continuity::get_cached(board_id, !use_cache) + .await + .unwrap() + .unwrap(); + log::info!( + "Downloaded continuity {board_id} - {}", + &continuity.board.name + ); + + log::info!("Caching all the icons..."); + continuity.cache_all_icons(false).await; + + for thread in &continuity.threads { + let name = thread_filename( + thread, + &continuity.board, + continuity.threads.iter().map(|t| t.post.section.clone()), + output_dir_layout, + ); + + if output_format.html() { + log::info!("Generating html document {name}..."); + let path = html_output_dir.join(format!("{name}.html")); + write(path, thread.to_single_html_page(html_options)); + } + + if output_format.epub() { + log::info!("Generating epub document {name}..."); + let path = epub_output_dir.join(format!("{name}.epub")); + write(path, thread.to_epub(epub_options).await.unwrap()); + } + } + } + Command::Board { + board_id, + single_file: true, + .. + } => { + if output_format.html() { + log::warn!("HTML output is not supported in single-file mode."); + if output_format == OutputFormat::Html { + return; + } + } + + log::info!("Downloading board/continuity {board_id}..."); + let continuity = Continuity::get_cached(board_id, !use_cache) + .await + .unwrap() + .unwrap(); + log::info!( + "Downloaded continuity {board_id} - {}", + &continuity.board.name + ); + + log::info!("Caching all the icons..."); + continuity.cache_all_icons(false).await; + + let name = { + let board_id = continuity.board.id; + let name = slug::slugify(&continuity.board.name); + format!("[{board_id}] {name}") + }; + + log::info!("Generating epub document {name}..."); + let path = epub_output_dir.join(format!("{name}.epub")); + write(path, continuity.to_epub(epub_options).await.unwrap()); + } + } + + log::info!("Done"); +} + +fn thread_filename( + thread: &Thread, + board: &Board, + board_thread_sections: impl Iterator>, + layout: OutputDirLayout, +) -> String { + let board_folder = { + let board_id = board.id; + let board_name = slug::slugify(&board.name); + format!("[{board_id}] {board_name}/") + }; + + let section_folder = thread + .post + .section + .clone() + .map(|Section { id, name, order }| { + let width = Ord::max(board.board_sections.len().to_string().len(), 2); + let name = slug::slugify(name); + format!("Section #{order:0width$} [{id}] {name}/") + }) + .unwrap_or_default(); + + let post_name = { + let post_id = thread.post.id; + let post_subject = slug::slugify(&thread.post.subject); + let post_order = { + let same_section_count = board_thread_sections + .filter(|s| *s == thread.post.section) + .count(); + let width = Ord::max(same_section_count.to_string().len(), 2); + let order = thread.post.section_order; + format!("{order:0width$}") + }; + format!("#{post_order} [{post_id}] {post_subject}") + }; + + format!("{board_folder}{section_folder}{post_name}").replace( + '/', + match layout { + OutputDirLayout::Flat => " ", + OutputDirLayout::Nested => "/", + }, + ) +} + +pub fn write(path: impl AsRef, contents: impl AsRef<[u8]>) { + std::fs::create_dir_all(path.as_ref().parent().unwrap()).unwrap(); + write_if_changed(path, contents).unwrap(); +} diff --git a/src/generate/epub.rs b/src/generate/epub.rs index 2b2f860..76c34fb 100644 --- a/src/generate/epub.rs +++ b/src/generate/epub.rs @@ -10,6 +10,7 @@ use uuid::Uuid; use crate::{ Board, Post, Reply, Thread, + intern_images::InternedImage, types::{Continuity, Section, User}, }; @@ -18,9 +19,20 @@ use super::{ }; impl Continuity { + #[cfg(not(target_arch = "wasm32"))] pub async fn to_epub(&self, options: Options) -> Result, Box> { - let mut images_to_intern = self.images_to_intern().await?; + let images_to_intern = self.images_to_intern().await?; + self.to_epub_from_images(options, images_to_intern) + } + /// Like [Self::to_epub], but with the images provided by the caller + /// instead of downloaded here. Available on wasm32, where the caller is + /// responsible for fetching the images (e.g. via JS `fetch`). + pub fn to_epub_from_images( + &self, + options: Options, + mut images_to_intern: HashMap, + ) -> Result, Box> { if let Some(size) = options.resize_icons { images_to_intern = images_to_intern .into_iter() @@ -382,9 +394,20 @@ impl Thread { } impl Thread { + #[cfg(not(target_arch = "wasm32"))] pub async fn to_epub(&self, options: Options) -> Result, Box> { - let mut images_to_intern = self.images_to_intern().await?; + let images_to_intern = self.images_to_intern().await?; + self.to_epub_from_images(options, images_to_intern) + } + /// Like [Self::to_epub], but with the images provided by the caller + /// instead of downloaded here. Available on wasm32, where the caller is + /// responsible for fetching the images (e.g. via JS `fetch`). + pub fn to_epub_from_images( + &self, + options: Options, + mut images_to_intern: HashMap, + ) -> Result, Box> { if let Some(size) = options.resize_icons { images_to_intern = images_to_intern .into_iter() diff --git a/src/intern_images.rs b/src/intern_images.rs index 0700e73..07193a3 100644 --- a/src/intern_images.rs +++ b/src/intern_images.rs @@ -1,17 +1,15 @@ -use std::{ - collections::{HashMap, HashSet}, - error::Error, - io::Cursor, -}; +#[cfg(not(target_arch = "wasm32"))] +use std::collections::{HashMap, HashSet}; +use std::{error::Error, io::Cursor}; use image::imageops::FilterType; use mime::Mime; -use crate::{ - cached::download_cached_image, - types::{Continuity, Icon, Thread}, - utils::{mime_to_image_extension, url_hash}, -}; +#[cfg(not(target_arch = "wasm32"))] +use crate::cached::download_cached_image; +#[cfg(not(target_arch = "wasm32"))] +use crate::types::{Continuity, Icon, Thread}; +use crate::utils::{mime_to_image_extension, url_hash}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct InternedImage { @@ -142,6 +140,7 @@ impl InternedImage { } } +#[cfg(not(target_arch = "wasm32"))] impl Continuity { pub async fn images_to_intern(&self) -> Result, Box> { let mut interned_images: HashMap = HashMap::new(); @@ -161,6 +160,7 @@ impl Continuity { } } +#[cfg(not(target_arch = "wasm32"))] impl Thread { pub async fn images_to_intern(&self) -> Result, Box> { let mut interned_images: HashMap = HashMap::new(); @@ -242,6 +242,7 @@ impl Thread { } } +#[cfg(not(target_arch = "wasm32"))] impl Icon { async fn intern(&self) -> Result> { let (mime, data) = self.download_cached(false).await?; diff --git a/src/lib.rs b/src/lib.rs index 4454ec6..e87eab2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,11 @@ mod rfc3339; +#[cfg(not(target_arch = "wasm32"))] mod auth; +#[cfg(not(target_arch = "wasm32"))] pub mod api; +#[cfg(not(target_arch = "wasm32"))] pub mod cached; pub mod generate; pub mod intern_images; diff --git a/src/main.rs b/src/main.rs index aa0883c..a0f644a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,376 +1,15 @@ -use clap::Parser; -use std::{ - fmt::Display, - path::{Path, PathBuf}, -}; +// The CLI is native-only; on wasm32 the binary is an empty stub so that +// whole-workspace builds (and CI) can target wasm32-unknown-unknown. +// Use the library's `generate` module for wasm. -use glowpub::{ - Board, Thread, - api::BoardPosts, - cached::write_if_changed, - generate::Options, - types::{Continuity, Section}, -}; - -const DEFAULT_OUTPUT_DIR: &str = "./books"; - -/// Download and process Glowfic posts into epub and html files. -#[derive(Debug, Parser)] -enum Command { - /// Download and process a single post. - Post { - /// The id of the Glowfic post. - /// Can be found in the URL: https://glowfic.com/posts/ - post_id: u64, - - #[command(flatten)] - options: CliOptions, - }, - /// Download and process an entire board. - Board { - /// The id of the Glowfic board. - /// Can be found in the URL: https://glowfic.com/boards/ - board_id: u64, - - #[command(flatten)] - options: CliOptions, - - /// If enabled, the board will be processed into a single epub file instead of being split by post. - #[clap(long)] - single_file: bool, - }, -} -impl Command { - fn options(&self) -> CliOptions { - match self { - Command::Post { options, .. } | Command::Board { options, .. } => options.clone(), - } - } -} - -#[derive(Debug, Clone, Parser)] -struct CliOptions { - /// Reuse already downloaded data. Images are always cached. - #[clap(long)] - use_cache: bool, - - /// Simplify character and user names to improve text-to-speech output. - #[clap(long)] - text_to_speech: bool, - - ///
tags can be hard to use on e-readers, this option forces them to always seem open. - /// - /// (Under the hood, it replaces the
tag with a
, and with

, - /// it also preprends `▼ ` to the

tag to make it similar to an open
tag.) - #[clap(long)] - flatten_details: Option, - - /// When inlining the images into the epub file, this will convert all images into jpeg files. - /// In general this will result in considerably smaller files if the images are not already jpegs. - /// (Does not affect SVGs.) - #[clap(long)] - jpeg: bool, - - /// When inlining icons into the epub file, this will scale all icon images above the provided width down to that width. - /// Defaults to "100" if no value is provided. - /// (Does not affect SVGs or non-icon images.) - #[clap(long)] - resize_icons: Option>, - - /// Output files in this directory (e.g. `--output-dir=~/glowfic`). - /// Note that this can flood the directory if used with `board` but without `--single-file`. - /// Files will be placed in format-specific subdirectories if this option is not set, or if `--output-format` is `both` (the default). - #[clap(long)] - output_dir: Option, - - /// Specify the output directory layout. - #[clap(long, default_value_t = OutputDirLayout::default())] - output_dir_layout: OutputDirLayout, - - /// Determines which file-types are created by the program. - #[clap(long, default_value_t = OutputFormat::default())] - output_format: OutputFormat, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] -enum FlattenDetails { - /// The default option. No
tags will be flattened. - #[default] - None, - /// All
tags will be flattened. - All, - /// Only
tags in epubs will be flattened. - Mixed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] -enum OutputDirLayout { - /// The default option. Output files will be placed in a nested subdirectory based on their board. - #[default] - Nested, - /// Output files will be placed directly in the output directory, and will have their board information included in the filename. - Flat, -} -impl Display for OutputDirLayout { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Nested => write!(f, "nested"), - Self::Flat => write!(f, "flat"), - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] -enum OutputFormat { - /// The default option. Both HTML and Epub files will be created. - /// Output files will be placed in "html" and "epub" subdirectories, respectively. - #[default] - Both, - /// Only Epub files will be created. - Epub, - /// Only HTML files will be created. - Html, - /// No output files will be created. - None, -} -impl OutputFormat { - fn epub(self) -> bool { - matches!(self, Self::Epub | Self::Both) - } - fn html(self) -> bool { - matches!(self, Self::Html | Self::Both) - } -} -impl Display for OutputFormat { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Both => write!(f, "both"), - Self::Epub => write!(f, "epub"), - Self::Html => write!(f, "html"), - Self::None => write!(f, "none"), - } - } -} +#[cfg(not(target_arch = "wasm32"))] +mod cli; +#[cfg(not(target_arch = "wasm32"))] #[tokio::main] async fn main() { - simple_logger::init_with_level(log::Level::Info).unwrap(); - - let command = Command::parse(); - - let CliOptions { - use_cache, - text_to_speech, - flatten_details, - jpeg, - resize_icons, - output_dir, - output_dir_layout, - output_format, - } = command.options(); - - let resize_icons = resize_icons.map(|r| r.unwrap_or(100)); - - let mut html_output_dir = output_dir - .clone() - .unwrap_or_else(|| PathBuf::from(DEFAULT_OUTPUT_DIR)); - let mut epub_output_dir = output_dir - .clone() - .unwrap_or_else(|| PathBuf::from(DEFAULT_OUTPUT_DIR)); - - if output_format == OutputFormat::Both || output_dir.is_none() { - html_output_dir = html_output_dir.join("html/"); - epub_output_dir = epub_output_dir.join("epub/"); - } - - let epub_options = Options { - text_to_speech, - flatten_details: match flatten_details.unwrap_or_default() { - FlattenDetails::All | FlattenDetails::Mixed => true, - FlattenDetails::None => false, - }, - jpeg, - resize_icons, - }; - let html_options = Options { - text_to_speech, - flatten_details: match flatten_details.unwrap_or_default() { - FlattenDetails::All => true, - FlattenDetails::None | FlattenDetails::Mixed => false, - }, - jpeg, - resize_icons, - }; - - match command { - Command::Post { post_id, .. } => { - log::info!("Downloading post {post_id}"); - let thread = Thread::get_cached(post_id, !use_cache) - .await - .unwrap() - .unwrap(); - log::info!("Downloaded post {post_id} - {}", &thread.post.subject); - - log::info!("Caching all the icons..."); - thread.cache_all_icons(false).await; - - let name = { - let board = Board::get_cached(thread.post.board.id, !use_cache) - .await - .unwrap() - .unwrap(); - - let board_posts = BoardPosts::get_all_cached(board.id, !use_cache) - .await - .unwrap() - .unwrap(); - - thread_filename( - &thread, - &board, - board_posts.iter().map(|p| p.section.clone()), - OutputDirLayout::Flat, - ) - }; - - if output_format.html() { - log::info!("Generating html document {name}..."); - let path = html_output_dir.join(format!("{name}.html")); - write(path, thread.to_single_html_page(html_options)); - } - - if output_format.epub() { - log::info!("Generating epub document {name}..."); - let path = epub_output_dir.join(format!("{name}.epub")); - write(path, thread.to_epub(epub_options).await.unwrap()); - } - } - Command::Board { - board_id, - single_file: false, - .. - } => { - log::info!("Downloading board/continuity {board_id}..."); - let continuity = Continuity::get_cached(board_id, !use_cache) - .await - .unwrap() - .unwrap(); - log::info!( - "Downloaded continuity {board_id} - {}", - &continuity.board.name - ); - - log::info!("Caching all the icons..."); - continuity.cache_all_icons(false).await; - - for thread in &continuity.threads { - let name = thread_filename( - thread, - &continuity.board, - continuity.threads.iter().map(|t| t.post.section.clone()), - output_dir_layout, - ); - - if output_format.html() { - log::info!("Generating html document {name}..."); - let path = html_output_dir.join(format!("{name}.html")); - write(path, thread.to_single_html_page(html_options)); - } - - if output_format.epub() { - log::info!("Generating epub document {name}..."); - let path = epub_output_dir.join(format!("{name}.epub")); - write(path, thread.to_epub(epub_options).await.unwrap()); - } - } - } - Command::Board { - board_id, - single_file: true, - .. - } => { - if output_format.html() { - log::warn!("HTML output is not supported in single-file mode."); - if output_format == OutputFormat::Html { - return; - } - } - - log::info!("Downloading board/continuity {board_id}..."); - let continuity = Continuity::get_cached(board_id, !use_cache) - .await - .unwrap() - .unwrap(); - log::info!( - "Downloaded continuity {board_id} - {}", - &continuity.board.name - ); - - log::info!("Caching all the icons..."); - continuity.cache_all_icons(false).await; - - let name = { - let board_id = continuity.board.id; - let name = slug::slugify(&continuity.board.name); - format!("[{board_id}] {name}") - }; - - log::info!("Generating epub document {name}..."); - let path = epub_output_dir.join(format!("{name}.epub")); - write(path, continuity.to_epub(epub_options).await.unwrap()); - } - } - - log::info!("Done"); + cli::main().await } -fn thread_filename( - thread: &Thread, - board: &Board, - board_thread_sections: impl Iterator>, - layout: OutputDirLayout, -) -> String { - let board_folder = { - let board_id = board.id; - let board_name = slug::slugify(&board.name); - format!("[{board_id}] {board_name}/") - }; - - let section_folder = thread - .post - .section - .clone() - .map(|Section { id, name, order }| { - let width = Ord::max(board.board_sections.len().to_string().len(), 2); - let name = slug::slugify(name); - format!("Section #{order:0width$} [{id}] {name}/") - }) - .unwrap_or_default(); - - let post_name = { - let post_id = thread.post.id; - let post_subject = slug::slugify(&thread.post.subject); - let post_order = { - let same_section_count = board_thread_sections - .filter(|s| *s == thread.post.section) - .count(); - let width = Ord::max(same_section_count.to_string().len(), 2); - let order = thread.post.section_order; - format!("{order:0width$}") - }; - format!("#{post_order} [{post_id}] {post_subject}") - }; - - format!("{board_folder}{section_folder}{post_name}").replace( - '/', - match layout { - OutputDirLayout::Flat => " ", - OutputDirLayout::Nested => "/", - }, - ) -} - -pub fn write(path: impl AsRef, contents: impl AsRef<[u8]>) { - std::fs::create_dir_all(path.as_ref().parent().unwrap()).unwrap(); - write_if_changed(path, contents).unwrap(); -} +#[cfg(target_arch = "wasm32")] +fn main() {} diff --git a/src/utils.rs b/src/utils.rs index 2571d2a..0f31b60 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,6 @@ -use std::{io::Cursor, str::FromStr, sync::OnceLock}; +use std::{io::Cursor, str::FromStr}; +#[cfg(not(target_arch = "wasm32"))] +use std::sync::OnceLock; use image::ImageReader; use mime::Mime; @@ -6,6 +8,7 @@ use sha2::{Digest, Sha256}; use crate::types::{Icon, Thread}; +#[cfg(not(target_arch = "wasm32"))] const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),); impl Thread { @@ -59,6 +62,7 @@ pub fn url_hash(url: &str) -> String { format!("{hash:x}") } +#[cfg(not(target_arch = "wasm32"))] pub fn http_client() -> reqwest::Client { // TODO: use global `std::sync::LazyLock` once stable. pub static HTTP_CLIENT: OnceLock = OnceLock::new();