use std::collections::HashSet; use std::{borrow::Cow, collections::BTreeMap, error::Error, time::Duration}; use chrono::NaiveDate; use futures::{future::ready, stream::iter, StreamExt, TryStreamExt}; use lofty::file::{AudioFile, TaggedFileExt}; use lofty::tag::Accessor; use log::info; use regex::Regex; use render::Renderer; use serde::{Deserialize, Serialize}; use str_slug::slug; pub mod de; mod render; pub mod s3; /// The Index tracks the state from the last successful exection. #[derive(Default, Serialize, Deserialize)] pub struct Index { #[serde(default)] entries: BTreeMap, #[serde(default)] templates: BTreeMap, #[serde(default)] rendered: BTreeMap>, } impl Index { pub fn contains_key(&self, key: &str) -> bool { self.entries.contains_key(key) } pub fn add(&mut self, entry: Entry) { self.entries.insert(entry.filename.to_owned(), entry); } pub fn remove(&mut self, filename: &str) { self.entries.remove(filename); } } /// Entry records the metadata about a file in the collection. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Entry { pub filename: String, pub date: Option, pub etag: Option, pub title: Option, pub album: Option, pub artist: Option, pub size: i64, #[serde(with = "de::hms_duration")] pub duration: Duration, pub hidden: bool, } impl Entry { pub fn read_from( key: String, etag: Option, size: i64, date: Option, mut file: std::fs::File, ) -> Result> { let tagged = lofty::read_from(&mut file)?; let tag = tagged.primary_tag(); Ok(Entry { filename: key.to_string(), etag, date, title: tag.and_then(|t| t.title()).map(Cow::into_owned), artist: tag.and_then(|t| t.artist()).map(Cow::into_owned), album: tag.and_then(|t| t.album()).map(Cow::into_owned), size, duration: tagged.properties().duration(), hidden: false, }) } } /// Templates are used to render content for the feed. /// /// `partial` templates are never rendered, but may be included in other templates. /// `index` templates are rendered once for the entire collection. /// Non-index templates are rendered once per entry. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Template { pub name: String, /// False if the template should be rendered for each entry. /// True if the template renders the list of all entries. #[serde(default)] pub index: bool, #[serde(default)] pub partial: bool, pub content_type: Option, #[serde(default)] /// Only render this template for files that match this regex. pub filter: Option, pub template: String, } impl Template { fn make_filename(&self, entry: &Entry) -> String { if self.index { self.name.to_string() } else { let (basename, extension) = self .name .rsplit_once(".") .unwrap_or_else(|| (&self.name, ".html")); let title = if let Some(title) = &entry.title { slug(title) } else { slug( entry .filename .rsplit_once(".") .unwrap_or_else(|| (&entry.filename, "")) .0, ) }; if let Some(ref date) = entry.date { format!("{basename}/{date}-{title}.{extension}") } else { format!("{basename}/{title}.{extension}") } } } } pub struct MP32RSS { access: s3::Access, index: Index, index_etag: Option, } impl MP32RSS { pub async fn open(access: s3::Access) -> Result> { info!("Opening index file"); let (index, index_etag) = access.fetch_index().await?; Ok(Self { access, index, index_etag, }) } pub fn get_mut_entry(&mut self, filename: &str) -> Option<&mut Entry> { self.index.entries.get_mut(filename) } pub async fn sync(&mut self) -> Result<(), Box> { info!("Saving index file"); let new_etag = self .access .put_index(&self.index, self.index_etag.as_deref()) .await?; self.index_etag = new_etag; Ok(()) } pub async fn refresh(&mut self) -> Result<(), Box> { info!("Syncing against files in bucket"); // 2. List files in the bucket let objects = self.access.list_mp3s().await?; // 3. Find files missing in the index // 4. For each missing file, download and add to local index let new_entries: Vec<_> = iter(objects) .filter(|k| ready(!self.index.contains_key(k))) .then(|k| { info!("Found new file: {k}..."); self.access.fetch_entry(k) }) .try_collect() .await?; let mut new_filenames = HashSet::new(); for entry in new_entries { new_filenames.insert(entry.filename.to_string()); self.index.add(entry); } // 5. Generate files for each template and upload // 6. Upload each file // Check the ETAG against the ETAG in the index object? self.render(|e| new_filenames.contains(&e.filename)).await?; // 7. Upload the index file // If upload fails, abort (or retry from the beginning) self.sync().await } pub async fn delete(&mut self, filename: &str) -> Result<(), Box> { self.index.remove(filename); self.access.remove_file(filename).await?; self.render(|e| e.filename == filename).await?; self.sync().await } pub fn templates(&self) -> impl Iterator { self.index.templates.values() } /// Add a new template and rerender. pub async fn add_template( &mut self, name: String, template: String, index: bool, partial: bool, content_type: Option, filter: Option, ) -> Result<(), Box> { self.index.templates.insert( name.to_owned(), Template { name, template, index, partial, content_type, filter, }, ); self.render(|_| true).await?; self.sync().await } /// Add a new template and rerender. pub async fn delete_template(&mut self, name: &str) -> Result<(), Box> { self.index.templates.remove(name); self.render(|_| true).await?; self.sync().await } /// Render all templates. /// // Filter limits rerendering to entries which return true. pub async fn render(&mut self, filter: F) -> Result<(), Box> where F: Fn(&Entry) -> bool, { let empty_set = HashSet::new(); let mut renderer = Renderer::new(self.index.templates.values())?; for template in self.index.templates.values() { let existing_files = self .index .rendered .get(&template.name) .unwrap_or(&empty_set); let mut new_files = HashSet::new(); if !template.partial { info!("Rendering template {}", template.name); let content_type = template.content_type.as_deref().unwrap_or("text/html"); let file_regex = Regex::new(template.filter.as_deref().unwrap_or(".*"))?; let entries: Vec<_> = self .index .entries .values() .filter(|e| !e.hidden && file_regex.is_match(&e.filename)) .cloned() .collect(); renderer.set_entries(entries.clone()); if template.index { let filename = template.name.to_string(); let data = renderer.render_index(&template.name, &filename)?; self.access.put_file(&filename, content_type, data).await?; new_files.insert(filename); } else { // Generate a file for each entry. for entry in &entries { let filename = template.make_filename(entry); if filter(entry) { let data = renderer.render_entry(&template.name, &filename, entry)?; self.access.put_file(&filename, content_type, data).await?; new_files.insert(filename); } else if existing_files.contains(&filename) { new_files.insert(filename); } } } } // Remove any orphaned files for filename in existing_files { if !new_files.contains(filename) { info!("Removing rendered file {}", filename); self.access.remove_file(filename).await?; } } // Update the list of rendered files. self.index .rendered .insert(template.name.to_string(), new_files); } // Remove renderings of any deleted templates for (template, files) in &self.index.rendered { if !self.index.templates.contains_key(template) { info!("Removing all rendered files for template {}", template); for filename in files { self.access.remove_file(filename).await?; } } } self.index .rendered .retain(|k, _| self.index.templates.contains_key(k)); self.sync().await } }