Compare commits

..

No commits in common. "d7882b029144f268a4201ba712cceb74148aec79" and "c8301d14e16db8de60fddbd05f37b1fc8c8eeab6" have entirely different histories.

6 changed files with 20 additions and 64 deletions

View File

@ -1,6 +1,6 @@
{
"db_name": "SQLite",
"query": "\n SELECT \n id as \"id!: String\",\n local_id as \"local_id!: String\",\n title,\n published as \"published: Option<chrono::DateTime<Utc>>\",\n updated as \"updated: Option<chrono::DateTime<Utc>>\",\n summary,\n content,\n link,\n marked_read as \"marked_read: Option<chrono::DateTime<Utc>>\"\n FROM feed_entries\n WHERE feed_id = ?\n ORDER BY published DESC NULLS LAST\n LIMIT ?\n ",
"query": "\n SELECT \n id as \"id!: String\",\n local_id as \"local_id!: String\",\n title,\n published as \"published: Option<chrono::DateTime<Utc>>\",\n updated as \"updated: Option<chrono::DateTime<Utc>>\",\n summary,\n content,\n link,\n marked_read as \"marked_read: Option<chrono::DateTime<Utc>>\"\n FROM feed_entries \n WHERE feed_id = ?\n ORDER BY published DESC NULLS LAST\n LIMIT ?\n ",
"describe": {
"columns": [
{
@ -64,5 +64,5 @@
true
]
},
"hash": "6186c55b12f42931e31a9f8367d6aa2bd637091ca9e1b8d9459241ca9488882c"
"hash": "2bbed6f20243ced25ac9359afefafb5ddffdff949250e0e4e35bf399fc0199fc"
}

14
Cargo.lock generated
View File

@ -923,7 +923,6 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
@ -974,17 +973,6 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-macro"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.96",
]
[[package]]
name = "futures-sink"
version = "0.3.31"
@ -1006,7 +994,6 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
@ -2681,7 +2668,6 @@ dependencies = [
"chrono",
"clap",
"feed-rs",
"futures",
"getrandom 0.2.15",
"reqwest",
"rocket",

View File

@ -24,4 +24,3 @@ base64 = "0.21"
getrandom = "0.2"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
futures = "0.3.31"

View File

@ -1,7 +1,6 @@
use crate::feeds::Feed;
use crate::poll_utils::{fetch_new_entries, update_entry_db};
use crate::poll_utils::fetch_new_entries;
use crate::user::User;
use futures::future::join_all;
use tracing::{info, warn};
struct DemoFeed {
@ -10,7 +9,7 @@ struct DemoFeed {
category: Option<&'static str>,
}
const DEMO_FEEDS: [DemoFeed; 6] = [
const DEMO_FEEDS: [DemoFeed; 5] = [
DemoFeed {
name: "XKCD",
url: "https://xkcd.com/atom.xml",
@ -31,11 +30,6 @@ const DEMO_FEEDS: [DemoFeed; 6] = [
url: "https://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml",
category: Some("News"),
},
DemoFeed {
name: "Lines and Colors",
url: "https://linesandcolors.com/feed/",
category: None,
},
DemoFeed {
name: "Astronomy Picture of the Day (APOD)",
url: "https://apod.nasa.gov/apod.rss",
@ -89,36 +83,17 @@ pub async fn setup_demo_data(pool: &sqlx::SqlitePool) {
.expect("Failed to create demo feed");
}
// Fetch all feeds in parallel
let fetch_futures: Vec<_> = feeds
.iter()
.map(|feed| {
let url = feed.url.clone();
let feed_id = feed.feed_id;
async move {
let result = fetch_new_entries(&url).await;
(feed_id, url, result)
}
})
.collect();
for feed in feeds.iter() {
let url = &feed.url;
let results = join_all(fetch_futures).await;
for (feed_id, url, result) in results {
match result {
Ok(entries) => {
info!(
feed_url = url.as_str(),
"Successfully fetched {} entries",
entries.len()
);
if let Err(e) = update_entry_db(&entries, &feed_id, pool).await {
warn!(error=%e, feed_url=url.as_str(), "Failed to store entries in database");
}
}
let entries = match fetch_new_entries(url).await {
Ok(entries) => entries,
Err(e) => {
warn!(error=%e, feed_url=url.as_str(), "Error populating feed");
continue;
}
}
};
//update_entry_db(&entries, &feed_id, &mut db).await?;
}
info!("Successfully set up demo data");

View File

@ -27,7 +27,6 @@ pub struct EntryStateUpdate {
}
async fn read_entries(feed_id: &Uuid, db: &mut SqliteConnection) -> Result<Vec<Entry>, Status> {
let feed_id_str = feed_id.to_string();
let rows = sqlx::query!(
r#"
SELECT
@ -40,12 +39,12 @@ async fn read_entries(feed_id: &Uuid, db: &mut SqliteConnection) -> Result<Vec<E
content,
link,
marked_read as "marked_read: Option<chrono::DateTime<Utc>>"
FROM feed_entries
FROM feed_entries
WHERE feed_id = ?
ORDER BY published DESC NULLS LAST
LIMIT ?
"#,
feed_id_str,
feed_id,
MAX_ENTRIES_PER_FEED,
)
.fetch_all(db)
@ -108,11 +107,11 @@ pub async fn poll_feed(
info!("Feed {} last checked: {}", feed_id, last_checked);
let entries = if last_checked < POLLING_INTERVAL {
info!("Reading entries from database for feed {}", feed_id);
read_entries(&feed_id, &mut **db).await?
read_entries(&feed_id, &mut db).await?
} else {
info!("Fetching new entries for feed {}", feed_id);
let entries = crate::poll_utils::fetch_new_entries(&url).await?;
update_entry_db(&entries, &feed_id, &mut **db).await?;
update_entry_db(&entries, &feed_id, &mut db).await?;
entries
};

View File

@ -3,7 +3,7 @@ use chrono::{DateTime, Utc};
use feed_rs::model::Text;
use rocket::http::Status;
use rocket::serde::{json, uuid::Uuid, Serialize};
use sqlx::{Acquire, Executor};
use sqlx::{Acquire, SqliteConnection};
use url::Url;
#[derive(Debug, Serialize)]
@ -50,16 +50,13 @@ pub async fn fetch_new_entries(url: &Url) -> Result<Vec<Entry>, Status> {
Ok(entries)
}
pub async fn update_entry_db<'a, E>(
pub async fn update_entry_db(
entries: &Vec<Entry>,
feed_id: &Uuid,
executor: E,
) -> Result<(), Status>
where
E: Executor<'a, Database = sqlx::Sqlite> + Acquire<'a, Database = sqlx::Sqlite>,
{
db: &mut SqliteConnection,
) -> Result<(), Status> {
// Start a transaction for batch update
let mut tx = executor.begin().await.map_err(|e| {
let mut tx = db.begin().await.map_err(|e| {
error!("Failed to start transaction: {}", e);
Status::InternalServerError
})?;