documents: make "sync to this node" recurse into child documents
Build Debug APK / build (push) Failing after 19m5s
Build Debug APK / build (push) Failing after 19m5s
Syncing a folder/collection now pulls blobs and artifact values for the whole subtree, not just the top-level document. Adds asset_sync::sync_document_assets_recursive (walks list_children, merges per-document DocAssetSyncResult with a new documents_synced count) and threads it through the FFI, the gpui and Android document-detail "Sync to this node" controls (both now show the descendant count and mention the recursion), and a new `syn-cli docs sync` subcommand. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -306,9 +306,10 @@ class NativeLib {
|
||||
|
||||
fun syncDocuments(nodeIdHex: String): Result<UInt> = ffi { node().syncDocuments(nodeIdHex) }
|
||||
|
||||
// Pull this document's blobs + artifact values from ensemble peers into the
|
||||
// local node, persisting everything fetched. Backs the document-detail
|
||||
// "Sync to this node" control.
|
||||
// Pull this document's blobs + artifact values, and those of every descendant
|
||||
// document, from ensemble peers into the local node, persisting everything
|
||||
// fetched. Backs the document-detail "Sync to this node" control, which
|
||||
// recurses into the whole subtree.
|
||||
fun syncDocumentAssets(documentId: String): Result<DocumentSyncResult> = ffi {
|
||||
node().syncDocumentAssets(documentId)
|
||||
}
|
||||
|
||||
+28
-6
@@ -36,6 +36,9 @@ fun DocumentDetailScreen(documentId: String, onBack: () -> Unit, modifier: Modif
|
||||
var blobRefs by remember { mutableStateOf<List<DocumentBlobRefItem>>(emptyList()) }
|
||||
var nodeStatuses by remember { mutableStateOf<List<DocumentNodeStatusItem>>(emptyList()) }
|
||||
var error by remember { mutableStateOf<String?>(null) }
|
||||
// Descendant count shown on the Sync card so users know syncing recurses
|
||||
// into children; null while still loading.
|
||||
var descendantCount by remember { mutableStateOf<UInt?>(null) }
|
||||
|
||||
// Bumped after a sync so the LaunchedEffect re-reads node-presence badges and
|
||||
// blob availability for the freshly-pulled assets.
|
||||
@@ -74,6 +77,10 @@ fun DocumentDetailScreen(documentId: String, onBack: () -> Unit, modifier: Modif
|
||||
fail(it)
|
||||
return@LaunchedEffect
|
||||
}
|
||||
nativeLib
|
||||
.countDocumentDescendants(documentId)
|
||||
.onSuccess { count -> descendantCount = count }
|
||||
.onFailure { Log.w(TAG, "Failed to count descendants for $documentId", it) }
|
||||
}
|
||||
|
||||
val title = document?.title?.ifEmpty { "Untitled Document" } ?: "Document Details"
|
||||
@@ -114,6 +121,7 @@ fun DocumentDetailScreen(documentId: String, onBack: () -> Unit, modifier: Modif
|
||||
SyncCard(
|
||||
syncing = syncing,
|
||||
status = syncStatus,
|
||||
descendantCount = descendantCount,
|
||||
onSync = {
|
||||
syncing = true
|
||||
syncStatus = "Syncing…"
|
||||
@@ -133,8 +141,9 @@ fun DocumentDetailScreen(documentId: String, onBack: () -> Unit, modifier: Modif
|
||||
"Already up to date on this node"
|
||||
} else {
|
||||
"Pulled ${r.blobsReceived} blob(s), " +
|
||||
"${r.artifactsReceived} artifact(s) from " +
|
||||
"${r.peersTried} ensemble peer(s)"
|
||||
"${r.artifactsReceived} artifact(s) " +
|
||||
"across ${r.documentsSynced} document(s) " +
|
||||
"from ${r.peersTried} ensemble peer(s)"
|
||||
}
|
||||
// Refresh presence/availability badges.
|
||||
reloadKey += 1
|
||||
@@ -266,7 +275,12 @@ private fun MetadataCard(
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun SyncCard(syncing: Boolean, status: String?, onSync: () -> Unit) {
|
||||
private fun SyncCard(
|
||||
syncing: Boolean,
|
||||
status: String?,
|
||||
descendantCount: UInt?,
|
||||
onSync: () -> Unit,
|
||||
) {
|
||||
Card(modifier = Modifier.fillMaxWidth()) {
|
||||
Column(
|
||||
modifier = Modifier.padding(16.dp),
|
||||
@@ -298,10 +312,18 @@ private fun SyncCard(syncing: Boolean, status: String?, onSync: () -> Unit) {
|
||||
)
|
||||
}
|
||||
}
|
||||
val helpText =
|
||||
if (descendantCount != null && descendantCount > 0u) {
|
||||
"Pulls this document's missing blobs and artifacts, and those of its " +
|
||||
"$descendantCount descendant document(s), from other nodes in your " +
|
||||
"ensemble that have them."
|
||||
} else {
|
||||
"Pulls this document's missing blobs and artifacts (recursively, for " +
|
||||
"any child documents) from other nodes in your ensemble that have " +
|
||||
"them."
|
||||
}
|
||||
Text(
|
||||
text =
|
||||
"Pulls this document's missing blobs and artifacts from other nodes " +
|
||||
"in your ensemble that have them.",
|
||||
text = helpText,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
|
||||
@@ -298,6 +298,17 @@ pub(crate) enum DocsCmd {
|
||||
#[bpaf(positional("ID"))]
|
||||
id: String,
|
||||
},
|
||||
|
||||
/// Pull a document's missing blobs and artifact values to this node from
|
||||
/// ensemble peers. Recurses into every descendant document (children,
|
||||
/// grandchildren, ...), so syncing a folder or collection materializes
|
||||
/// its whole subtree.
|
||||
#[bpaf(command("sync"))]
|
||||
Sync {
|
||||
/// Document ID
|
||||
#[bpaf(positional("ID"))]
|
||||
id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Bpaf, Debug, Clone)]
|
||||
|
||||
@@ -24,6 +24,25 @@ pub fn format_response(cmd: &DaemonCommand, value: Value) -> String {
|
||||
};
|
||||
format!("Purged {count} document(s); {broadcast_note}")
|
||||
}
|
||||
DaemonCommand::SyncDocumentAssets { .. } => {
|
||||
let blobs_requested = value["blobs_requested"].as_u64().unwrap_or(0);
|
||||
let blobs_received = value["blobs_received"].as_u64().unwrap_or(0);
|
||||
let artifacts_requested = value["artifacts_requested"].as_u64().unwrap_or(0);
|
||||
let artifacts_received = value["artifacts_received"].as_u64().unwrap_or(0);
|
||||
let documents_synced = value["documents_synced"].as_u64().unwrap_or(0);
|
||||
let peers_tried = value["peers_tried"].as_u64().unwrap_or(0);
|
||||
if blobs_requested == 0 && artifacts_requested == 0 {
|
||||
format!(
|
||||
"Already up to date on this node ({documents_synced} document(s) checked, recursively)"
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"Pulled {blobs_received} blob(s), {artifacts_received} artifact(s) across \
|
||||
{documents_synced} document(s) (recursively) from {peers_tried} ensemble \
|
||||
peer(s)"
|
||||
)
|
||||
}
|
||||
}
|
||||
DaemonCommand::Notebook(sub) => format_notebook(sub, &value),
|
||||
DaemonCommand::Note(sub) => format_note(sub, &value),
|
||||
DaemonCommand::Collection(sub) => format_collection(sub, &value),
|
||||
|
||||
@@ -140,6 +140,7 @@ fn build_command(cmd: cli::AgentCmd) -> Result<DaemonCommand> {
|
||||
DocsCmd::Show { .. } => unreachable!("Show is handled before daemon dispatch"),
|
||||
DocsCmd::Delete { id } => Ok(DaemonCommand::DeleteDocument { id: parse_id(&id)? }),
|
||||
DocsCmd::Purge { .. } => unreachable!("Purge is handled before daemon dispatch"),
|
||||
DocsCmd::Sync { id } => Ok(DaemonCommand::SyncDocumentAssets { id: parse_id(&id)? }),
|
||||
},
|
||||
|
||||
AgentCmd::Notes(NotesCmd::Notebook { sub }) => Ok(DaemonCommand::Notebook(match sub {
|
||||
|
||||
@@ -22,7 +22,7 @@ use synchronicity::documents::{
|
||||
Document, DocumentID, DocumentType, Property, PropertyKind, PropertyValue,
|
||||
};
|
||||
use synchronicity::node::{NodePubkey, NodeRuntime};
|
||||
use synchronicity::p2p::asset_sync::sync_document_assets;
|
||||
use synchronicity::p2p::asset_sync::sync_document_assets_recursive;
|
||||
use synchronicity::stores::Store;
|
||||
use synchronicity::stores::document::{DEFAULT_DOCUMENT_PAGE_SIZE, DocumentCursor, SearchParams};
|
||||
use tokio::runtime::Handle;
|
||||
@@ -70,6 +70,9 @@ struct Detail {
|
||||
state: DetailState,
|
||||
sync: SyncState,
|
||||
purge: PurgeState,
|
||||
/// Number of descendants (children, grandchildren, ...) this document has,
|
||||
/// shown on the Sync card since syncing recurses into the whole subtree.
|
||||
descendant_count: usize,
|
||||
/// The document's properties, rendered as a reusable [`Table`]. Populated
|
||||
/// once detail data loads.
|
||||
properties_table: Entity<Table>,
|
||||
@@ -532,6 +535,11 @@ impl DocumentsView {
|
||||
return;
|
||||
};
|
||||
let properties_table = cx.new(|_| Table::new(property_columns(), "No properties"));
|
||||
let descendant_count =
|
||||
self.store.doc_store.count_descendants(&doc_id).unwrap_or_else(|e| {
|
||||
log::error!("Failed to count descendants for {doc_id}: {e}");
|
||||
0
|
||||
});
|
||||
self.detail = Some(Detail {
|
||||
id: doc_id,
|
||||
title: row.title.clone(),
|
||||
@@ -540,6 +548,7 @@ impl DocumentsView {
|
||||
state: DetailState::Loading,
|
||||
sync: SyncState::Idle,
|
||||
purge: PurgeState::Idle,
|
||||
descendant_count,
|
||||
properties_table,
|
||||
});
|
||||
self.load_detail(cx);
|
||||
@@ -590,7 +599,8 @@ impl DocumentsView {
|
||||
.detach();
|
||||
}
|
||||
|
||||
/// Pull this document's missing blobs and artifacts from ensemble peers.
|
||||
/// Pull this document's missing blobs and artifacts from ensemble peers,
|
||||
/// recursing into every descendant document.
|
||||
fn sync_assets(&mut self, cx: &mut Context<Self>) {
|
||||
let Some(detail) = self.detail.as_mut() else {
|
||||
return;
|
||||
@@ -626,14 +636,20 @@ impl DocumentsView {
|
||||
let peer_ids: Vec<String> = peers.iter().map(|p| p.node_id_hex()).collect();
|
||||
let peer_count = peer_ids.len();
|
||||
let res = rt
|
||||
.block_on(sync_document_assets(&endpoint, &store, &doc_id, &peer_ids))
|
||||
.block_on(sync_document_assets_recursive(
|
||||
&endpoint, &store, &doc_id, &peer_ids,
|
||||
))
|
||||
.map_err(|e| e.to_string())?;
|
||||
if res.blobs_requested == 0 && res.artifacts_requested == 0 {
|
||||
Ok("Already up to date on this node".to_string())
|
||||
} else {
|
||||
Ok(format!(
|
||||
"Pulled {} blob(s), {} artifact(s) from {} of {} ensemble peer(s)",
|
||||
res.blobs_received, res.artifacts_received, res.peers_tried, peer_count
|
||||
"Pulled {} blob(s), {} artifact(s) across {} document(s) from {} of {} ensemble peer(s)",
|
||||
res.blobs_received,
|
||||
res.artifacts_received,
|
||||
res.documents_synced,
|
||||
res.peers_tried,
|
||||
peer_count
|
||||
))
|
||||
}
|
||||
})
|
||||
@@ -977,8 +993,18 @@ impl DocumentsView {
|
||||
}),
|
||||
)
|
||||
.child(div().text_size(px(14.)).text_color(theme.text_secondary).child(
|
||||
"Pulls this document's missing blobs and artifacts from other nodes in your \
|
||||
ensemble that have them.",
|
||||
if detail.descendant_count > 0 {
|
||||
format!(
|
||||
"Pulls this document's missing blobs and artifacts, and those of its \
|
||||
{} descendant document(s), from other nodes in your ensemble that \
|
||||
have them.",
|
||||
detail.descendant_count
|
||||
)
|
||||
} else {
|
||||
"Pulls this document's missing blobs and artifacts (recursively, for any \
|
||||
child documents) from other nodes in your ensemble that have them."
|
||||
.to_string()
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
+12
-6
@@ -945,6 +945,9 @@ pub struct DocumentSyncResult {
|
||||
pub artifacts_received: u32,
|
||||
/// Number of peers contacted before the document was satisfied.
|
||||
pub peers_tried: u32,
|
||||
/// Number of documents covered by this sync: the requested document plus
|
||||
/// every descendant it recursed into.
|
||||
pub documents_synced: u32,
|
||||
}
|
||||
|
||||
// ── SyncNode: holds all per-node FFI state ────────────────────────────────────
|
||||
@@ -2125,14 +2128,16 @@ impl SyncNode {
|
||||
})
|
||||
}
|
||||
|
||||
/// Pull this document's blobs and artifact values from ensemble peers into the
|
||||
/// local node, persisting everything that is fetched.
|
||||
/// Pull this document's blobs and artifact values — and those of every
|
||||
/// descendant document (children, grandchildren, ...) — from ensemble
|
||||
/// peers into the local node, persisting everything that is fetched.
|
||||
///
|
||||
/// Unlike [`fetch_document_blob_to_path`](Self::fetch_document_blob_to_path),
|
||||
/// which streams a single blob to a temp file for playback, this materializes
|
||||
/// the document permanently: every missing blob ciphertext is verified and
|
||||
/// written to the blob store, and every stubbed artifact value is fetched and
|
||||
/// merged (last-write-wins). It is the backing call for the document-detail
|
||||
/// the whole subtree permanently: every missing blob ciphertext is verified
|
||||
/// and written to the blob store, and every stubbed artifact value is
|
||||
/// fetched and merged (last-write-wins), for the requested document and all
|
||||
/// of its descendants. It is the backing call for the document-detail
|
||||
/// "sync to this node" control.
|
||||
pub fn sync_document_assets(
|
||||
&self,
|
||||
@@ -2162,7 +2167,7 @@ impl SyncNode {
|
||||
};
|
||||
|
||||
let res = rt_handle
|
||||
.block_on(crate::p2p::asset_sync::sync_document_assets(
|
||||
.block_on(crate::p2p::asset_sync::sync_document_assets_recursive(
|
||||
&endpoint, &store, &doc_id, &peer_ids,
|
||||
))
|
||||
.map_err(|e| SyncError::General { msg: e.to_string() })?;
|
||||
@@ -2173,6 +2178,7 @@ impl SyncNode {
|
||||
artifacts_requested: res.artifacts_requested,
|
||||
artifacts_received: res.artifacts_received,
|
||||
peers_tried: res.peers_tried,
|
||||
documents_synced: res.documents_synced,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -68,6 +68,21 @@ pub struct DocAssetSyncResult {
|
||||
pub artifacts_received: u32,
|
||||
/// Number of peers actually contacted before the document was satisfied.
|
||||
pub peers_tried: u32,
|
||||
/// Number of documents this pass covered (the root plus every descendant
|
||||
/// it recursed into). Always 1 for [`sync_document_assets`]; set by
|
||||
/// [`sync_document_assets_recursive`] to reflect the whole subtree.
|
||||
pub documents_synced: u32,
|
||||
}
|
||||
|
||||
impl DocAssetSyncResult {
|
||||
fn merge(&mut self, other: DocAssetSyncResult) {
|
||||
self.blobs_requested += other.blobs_requested;
|
||||
self.blobs_received += other.blobs_received;
|
||||
self.artifacts_requested += other.artifacts_requested;
|
||||
self.artifacts_received += other.artifacts_received;
|
||||
self.peers_tried = self.peers_tried.max(other.peers_tried);
|
||||
self.documents_synced += other.documents_synced;
|
||||
}
|
||||
}
|
||||
|
||||
/// Pull every missing blob and stubbed artifact value for `doc_id` from `peers`.
|
||||
@@ -93,6 +108,7 @@ pub async fn sync_document_assets(
|
||||
let mut result = DocAssetSyncResult {
|
||||
blobs_requested: missing_blobs.len() as u32,
|
||||
artifacts_requested: pending_artifacts.len() as u32,
|
||||
documents_synced: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -149,6 +165,32 @@ pub async fn sync_document_assets(
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Pull every missing blob and stubbed artifact value for `doc_id` **and every
|
||||
/// document reachable from it** (children, grandchildren, ...) from `peers`.
|
||||
///
|
||||
/// This is the recursive counterpart to [`sync_document_assets`] and is what
|
||||
/// backs the document-detail "sync to this node" control: syncing a folder or
|
||||
/// collection materializes its whole subtree, not just the folder's own
|
||||
/// metadata. Children are discovered locally via [`crate::stores::document::DocumentStore::list_children`]
|
||||
/// (mirroring [`crate::stores::document::DocumentStore::purge_recursive`]'s
|
||||
/// traversal), then each document in the subtree is synced independently in
|
||||
/// depth-first order; one document's peer failures don't block the rest.
|
||||
pub async fn sync_document_assets_recursive(
|
||||
endpoint: &Endpoint,
|
||||
store: &Store,
|
||||
doc_id: &DocumentID,
|
||||
peers: &[String],
|
||||
) -> Result<DocAssetSyncResult> {
|
||||
let mut result = sync_document_assets(endpoint, store, doc_id, peers).await?;
|
||||
let children = store.doc_store.list_children(doc_id)?;
|
||||
for child in children {
|
||||
let child_result =
|
||||
Box::pin(sync_document_assets_recursive(endpoint, store, &child.id, peers)).await?;
|
||||
result.merge(child_result);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// BLAKE3 ciphertext hashes of `doc_id`'s blobs whose bytes are not on disk.
|
||||
fn missing_blob_hashes(store: &Store, doc_id: &DocumentID) -> Result<Vec<[u8; 32]>> {
|
||||
let refs = store.doc_store.get_blob_refs(doc_id)?;
|
||||
|
||||
@@ -189,6 +189,9 @@ pub fn dispatch(cmd: &DaemonCommand, store: &Store) -> Result<Value> {
|
||||
DaemonCommand::PurgeDocument { .. } => {
|
||||
unreachable!("PurgeDocument is handled before dispatch")
|
||||
}
|
||||
DaemonCommand::SyncDocumentAssets { .. } => {
|
||||
unreachable!("SyncDocumentAssets is handled before dispatch")
|
||||
}
|
||||
DaemonCommand::Notebook(sub) => {
|
||||
let local_node = store
|
||||
.settings_store
|
||||
@@ -370,6 +373,44 @@ pub async fn dispatch_purge(
|
||||
}))
|
||||
}
|
||||
|
||||
/// Pull a document's (and all its descendants') missing blobs and artifact
|
||||
/// values from ensemble peers into this node. See
|
||||
/// [`synchronicity::p2p::asset_sync::sync_document_assets_recursive`] for the
|
||||
/// traversal and per-peer fetch logic.
|
||||
pub async fn dispatch_sync_assets(
|
||||
id: &synchronicity::documents::DocumentID,
|
||||
store: &Store,
|
||||
p2p: &P2PNode,
|
||||
) -> Result<Value> {
|
||||
let ensemble_pubkey = store
|
||||
.settings_store
|
||||
.load_ensemble_membership()?
|
||||
.ok_or_else(|| anyhow::anyhow!("not in ensemble"))?
|
||||
.ensemble_pubkey;
|
||||
let peers = store.peer_node_store.get_peer_nodes_for_ensemble(&ensemble_pubkey)?;
|
||||
if peers.is_empty() {
|
||||
anyhow::bail!("no peers available to sync from");
|
||||
}
|
||||
let peer_ids: Vec<String> = peers.iter().map(|p| p.node_id_hex()).collect();
|
||||
|
||||
let res = synchronicity::p2p::asset_sync::sync_document_assets_recursive(
|
||||
&p2p.endpoint(),
|
||||
p2p.store(),
|
||||
id,
|
||||
&peer_ids,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(json!({
|
||||
"blobs_requested": res.blobs_requested,
|
||||
"blobs_received": res.blobs_received,
|
||||
"artifacts_requested": res.artifacts_requested,
|
||||
"artifacts_received": res.artifacts_received,
|
||||
"peers_tried": res.peers_tried,
|
||||
"documents_synced": res.documents_synced,
|
||||
}))
|
||||
}
|
||||
|
||||
fn tree_node_to_json(node: &NoteTreeNode) -> Value {
|
||||
match node {
|
||||
NoteTreeNode::Note { id, title, format } => {
|
||||
|
||||
@@ -42,6 +42,12 @@ pub enum DaemonCommand {
|
||||
PurgeDocument {
|
||||
id: DocumentID,
|
||||
},
|
||||
/// Pull a document's missing blobs and artifact values — and those of
|
||||
/// every descendant document (children, grandchildren, ...) — from
|
||||
/// ensemble peers into this node. Requires a running P2P node.
|
||||
SyncDocumentAssets {
|
||||
id: DocumentID,
|
||||
},
|
||||
Notebook(NotebookCmd),
|
||||
Note(NoteCmd),
|
||||
Collection(CollectionCmd),
|
||||
|
||||
@@ -246,6 +246,17 @@ async fn handle_command(
|
||||
};
|
||||
}
|
||||
|
||||
// SyncDocumentAssets needs both the store and the running P2P node (to
|
||||
// fetch blobs/artifacts from ensemble peers).
|
||||
if let DaemonCommand::SyncDocumentAssets { id } = &cmd {
|
||||
return match commands::dispatch_sync_assets(id, &state.runtime.store, &state.runtime.p2p)
|
||||
.await
|
||||
{
|
||||
Ok(result) => Json(CommandResponse::success(result)),
|
||||
Err(e) => Json(CommandResponse::error(format!("{e:#}"))),
|
||||
};
|
||||
}
|
||||
|
||||
// SetNodeName updates settings AND tells gossip to re-announce with the new name.
|
||||
if let DaemonCommand::SetNodeName { ref name } = cmd {
|
||||
let result = commands::dispatch(&cmd, &state.runtime.store);
|
||||
|
||||
Reference in New Issue
Block a user