Optimize write artifacts to disk in watch mode

crates/artifact_content/src/file_system_state.rs

@@ -0,0 +1,207 @@
+use std::{
+    collections::{HashMap, HashSet},
+    path::PathBuf,
+};
+
+use crate::operation_text::hash;
+use common_lang_types::{
+    ArtifactFileName, ArtifactHash, ArtifactPathAndContent, FileContent, FileSystemOperation,
+    SelectableName, ServerObjectEntityName,
+};
+use isograph_config::PersistedDocumentsHashAlgorithm;
+
+#[derive(Debug, Clone, Default)]
+pub struct FileSystemState {
+    root_files: HashMap<ArtifactFileName, (FileContent, ArtifactHash)>,
+    nested_files: HashMap<
+        ServerObjectEntityName,
+        HashMap<SelectableName, HashMap<ArtifactFileName, (FileContent, ArtifactHash)>>,
+    >,
+}
+
+impl FileSystemState {
+    pub fn insert(&mut self, path_and_content: &ArtifactPathAndContent) {
+        let value = (
+            FileContent::from(path_and_content.file_content.clone()),
+            ArtifactHash::from(hash(
+                &path_and_content.file_content,
+                PersistedDocumentsHashAlgorithm::Md5,
+            )),
+        );
+
+        match &path_and_content.artifact_path.type_and_field {
+            Some(type_and_field) => {
+                self.nested_files
+                    .entry(type_and_field.parent_object_entity_name)
+                    .or_default()
+                    .entry(type_and_field.selectable_name)
+                    .or_default()
+                    .insert(path_and_content.artifact_path.file_name, value);
+            }
+            None => {
+                self.root_files
+                    .insert(path_and_content.artifact_path.file_name, value);
+            }
+        }
+    }
+
+    pub fn from_artifacts(artifacts: Vec<ArtifactPathAndContent>) -> Self {
+        let mut state = Self::default();
+        for artifact in artifacts {
+            state.insert(&artifact);
+        }
+        state
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.root_files.is_empty() && self.nested_files.is_empty()
+    }
+
+    // Computes filesystem operations needed to transform this state into the target state.
+    // Returns operations to create, update, and delete files and directories. Files are updated
+    // only when their hash differs. If the current state is empty, emits a DeleteDirectory
+    // operation for the artifact root followed by recreation of all files. Empty directories
+    // are automatically removed after their files are deleted.
+    pub fn diff(&self, new: &Self, artifact_directory: &PathBuf) -> Vec<FileSystemOperation> {
+        let mut operations: Vec<FileSystemOperation> = Vec::new();
+
+        if self.is_empty() {
+            operations.push(FileSystemOperation::DeleteDirectory(
+                artifact_directory.to_path_buf(),
+            ));
+
+            for (server_object, selectables) in &new.nested_files {
+                let server_object_path = artifact_directory.join(server_object.to_string());
+                operations.push(FileSystemOperation::CreateDirectory(
+                    server_object_path.clone(),
+                ));
+
+                for (selectable, files) in selectables {
+                    let selectable_path = server_object_path.join(selectable.to_string());
+                    operations.push(FileSystemOperation::CreateDirectory(
+                        selectable_path.clone(),
+                    ));
+
+                    for (file_name, (content, _)) in files {
+                        let file_path = selectable_path.join(file_name.to_string());
+                        operations.push(FileSystemOperation::WriteFile(file_path, content.clone()));
+                    }
+                }
+
+                for (file_name, (content, _)) in &new.root_files {
+                    let file_path = artifact_directory.join(file_name.to_string());
+                    operations.push(FileSystemOperation::WriteFile(file_path, content.clone()));
+                }
+            }
+
+            return operations;
+        }
+
+        let mut new_server_objects: HashSet<&ServerObjectEntityName> = HashSet::new();
+        let mut new_selectables: HashSet<(&ServerObjectEntityName, &SelectableName)> =
+            HashSet::new();
+
+        for (server_object, selectables) in &new.nested_files {
+            new_server_objects.insert(server_object);
+            let server_object_path = artifact_directory.join(server_object.to_string());
+
+            if !self.nested_files.contains_key(server_object) {
+                operations.push(FileSystemOperation::CreateDirectory(
+                    server_object_path.clone(),
+                ));
+            }
+
+            for (selectable, files) in selectables {
+                new_selectables.insert((server_object, selectable));
+                let selectable_path = server_object_path.join(selectable.to_string());
+
+                let should_create_dir = self
+                    .nested_files
+                    .get(server_object)
+                    .and_then(|s| s.get(selectable))
+                    .is_none();
+
+                if should_create_dir {
+                    operations.push(FileSystemOperation::CreateDirectory(
+                        selectable_path.clone(),
+                    ));
+                }
+
+                for (file_name, (new_content, new_hash)) in files {
+                    let file_path = selectable_path.join(file_name.to_string());
+
+                    let should_write = self
+                        .nested_files
+                        .get(server_object)
+                        .and_then(|s| s.get(selectable))
+                        .and_then(|f| f.get(file_name))
+                        .map(|(_, old_hash)| old_hash != new_hash)
+                        .unwrap_or(true);
+
+                    if should_write {
+                        operations.push(FileSystemOperation::WriteFile(
+                            file_path,
+                            new_content.clone(),
+                        ));
+                    }
+                }
+            }
+        }
+
+        for (file_name, (new_content, new_hash)) in &new.root_files {
+            let file_path = artifact_directory.join(file_name.to_string());
+
+            let should_write = self
+                .root_files
+                .get(file_name)
+                .map(|(_old_content, old_hash)| old_hash != new_hash)
+                .unwrap_or(true);
+
+            if should_write {
+                operations.push(FileSystemOperation::WriteFile(
+                    file_path,
+                    new_content.clone(),
+                ));
+            }
+        }
+
+        for (server_object, selectables) in &self.nested_files {
+            let server_object_path = artifact_directory.join(server_object.to_string());
+
+            for (selectable, files) in selectables {
+                let selectable_path = server_object_path.join(selectable.to_string());
+
+                for file_name in files.keys() {
+                    let exist_in_new = new
+                        .nested_files
+                        .get(server_object)
+                        .and_then(|s| s.get(selectable))
+                        .and_then(|f| f.get(file_name))
+                        .is_some();
+
+                    if !exist_in_new {
+                        let file_path = selectable_path.join(file_name.to_string());
+                        operations.push(FileSystemOperation::DeleteFile(file_path));
+                    }
+                }
+
+                if !new_selectables.contains(&(server_object, selectable)) {
+                    operations.push(FileSystemOperation::DeleteDirectory(selectable_path));
+                }
+            }
+
+            if !new_server_objects.contains(server_object) {
+                operations.push(FileSystemOperation::DeleteDirectory(server_object_path));
+            }
+        }
+
+        for file_name in self.root_files.keys() {
+            if !new.root_files.contains_key(file_name) {
+                let file_path = artifact_directory.join(file_name.to_string());
+                operations.push(FileSystemOperation::DeleteFile(file_path));
+            }
+        }
+
+        return operations;
+    }
+}

crates/artifact_content/src/lib.rs

@@ -1,16 +1,18 @@
 mod eager_reader_artifact;
 mod entrypoint_artifact;
+mod file_system_state;
 mod format_parameter_type;
 pub mod generate_artifacts;
 mod imperatively_loaded_fields;
 mod import_statements;
 mod iso_overload_file;
 mod normalization_ast_text;
-mod operation_text;
+pub mod operation_text;
 mod persisted_documents;
 mod raw_response_type;
 mod reader_ast;
 mod refetch_reader_artifact;
 mod ts_config;
 
+pub use file_system_state::FileSystemState;
 pub use generate_artifacts::get_artifact_path_and_content;

crates/artifact_content/src/operation_text.rs

@@ -68,7 +68,7 @@ pub(crate) fn generate_operation_text<'a, TNetworkProtocol: NetworkProtocol>(
     }
 }
 
-fn hash(data: &str, algorithm: PersistedDocumentsHashAlgorithm) -> String {
+pub fn hash(data: &str, algorithm: PersistedDocumentsHashAlgorithm) -> String {
     match algorithm {
         PersistedDocumentsHashAlgorithm::Md5 => {
             let mut md5 = Md5::new();

crates/common_lang_types/src/file_system_operation.rs

@@ -0,0 +1,11 @@
+use std::path::PathBuf;
+
+use crate::FileContent;
+
+#[derive(Debug, Clone)]
+pub enum FileSystemOperation {
+    DeleteDirectory(PathBuf),
+    CreateDirectory(PathBuf),
+    WriteFile(PathBuf, FileContent),
+    DeleteFile(PathBuf),
+}

crates/common_lang_types/src/lib.rs

@@ -1,6 +1,7 @@
 mod absolute_and_relative_path;
 mod diagnostic;
 mod entity_and_selectable_name;
+mod file_system_operation;
 mod location;
 mod path_and_content;
 mod selectable_name;
@@ -12,6 +13,7 @@ mod text_with_carats;
 pub use absolute_and_relative_path::*;
 pub use diagnostic::*;
 pub use entity_and_selectable_name::*;
+pub use file_system_operation::*;
 pub use location::*;
 pub use path_and_content::*;
 pub use selectable_name::*;

crates/common_lang_types/src/path_and_content.rs

@@ -1,5 +1,6 @@
 use crate::{ArtifactFileName, ParentObjectEntityNameAndSelectableName};
 
+#[derive(Debug, Clone)]
 pub struct FileContent(pub String);
 
 impl From<String> for FileContent {
@@ -31,3 +32,32 @@ pub struct ArtifactPath {
     pub type_and_field: Option<ParentObjectEntityNameAndSelectableName>,
     pub file_name: ArtifactFileName,
 }
+
+#[derive(Debug, Clone)]
+pub struct ArtifactHash(String);
+
+impl From<String> for ArtifactHash {
+    fn from(value: String) -> Self {
+        ArtifactHash(value)
+    }
+}
+
+impl std::fmt::Display for ArtifactHash {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl std::ops::Deref for ArtifactHash {
+    type Target = String;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl PartialEq for ArtifactHash {
+    fn eq(&self, other: &Self) -> bool {
+        self.0 == other.0
+    }
+}

crates/isograph_compiler/src/batch_compile.rs

@@ -1,13 +1,14 @@
 use std::{path::PathBuf, time::Duration};
 
 use crate::{
-    compiler_state::CompilerState, with_duration::WithDuration,
-    write_artifacts::write_artifacts_to_disk,
+    compiler_state::CompilerState,
+    with_duration::WithDuration,
+    write_artifacts::{apply_file_system_operations, get_file_system_operations},
 };
 use artifact_content::get_artifact_path_and_content;
 use colored::Colorize;
 use common_lang_types::{CurrentWorkingDirectory, DiagnosticVecResult};
-use isograph_schema::{IsographDatabase, NetworkProtocol};
+use isograph_schema::NetworkProtocol;
 use prelude::Postfix;
 use pretty_duration::pretty_duration;
 use tracing::{error, info};
@@ -24,8 +25,10 @@ pub fn compile_and_print<TNetworkProtocol: NetworkProtocol>(
     current_working_directory: CurrentWorkingDirectory,
 ) -> DiagnosticVecResult<()> {
     info!("{}", "Starting to compile.".cyan());
-    let state = CompilerState::new(config_location, current_working_directory)?;
-    print_result(WithDuration::new(|| compile::<TNetworkProtocol>(&state.db)))
+    let mut state = CompilerState::new(config_location, current_working_directory)?;
+    print_result(WithDuration::new(|| {
+        compile::<TNetworkProtocol>(&mut state)
+    }))
 }
 
 pub fn print_result(
@@ -79,16 +82,22 @@ fn print_stats(elapsed_time: Duration, stats: CompilationStats) {
 /// This the "workhorse" command of batch compilation.
 #[tracing::instrument(skip_all)]
 pub fn compile<TNetworkProtocol: NetworkProtocol>(
-    db: &IsographDatabase<TNetworkProtocol>,
+    state: &mut CompilerState<TNetworkProtocol>,
 ) -> DiagnosticVecResult<CompilationStats> {
     // Note: we calculate all of the artifact paths and contents first, so that writing to
     // disk can be as fast as possible and we minimize the chance that changes to the file
     // system occur while we're writing and we get unpredictable results.
-
+    let db = &state.db;
     let config = db.get_isograph_config();
     let (artifacts, stats) = get_artifact_path_and_content(db)?;
-    let total_artifacts_written =
-        write_artifacts_to_disk(artifacts, &config.artifact_directory.absolute_path)?;
+
+    let file_system_operations = get_file_system_operations(
+        artifacts,
+        &config.artifact_directory.absolute_path,
+        &mut state.file_system_state,
+    );
+
+    let total_artifacts_written = apply_file_system_operations(file_system_operations)?;
 
     CompilationStats {
         client_field_count: stats.client_field_count,

crates/isograph_compiler/src/compiler_state.rs

@@ -10,13 +10,15 @@ use pico::Database;
 use prelude::Postfix;
 
 use crate::source_files::initialize_sources;
+use artifact_content::FileSystemState;
 
 const GC_DURATION_SECONDS: u64 = 60;
 
 #[derive(Debug)]
 pub struct CompilerState<TNetworkProtocol: NetworkProtocol> {
     pub db: IsographDatabase<TNetworkProtocol>,
     pub last_gc_run: Instant,
+    pub file_system_state: FileSystemState,
 }
 
 impl<TNetworkProtocol: NetworkProtocol> CompilerState<TNetworkProtocol> {
@@ -31,6 +33,7 @@ impl<TNetworkProtocol: NetworkProtocol> CompilerState<TNetworkProtocol> {
         Self {
             db,
             last_gc_run: Instant::now(),
+            file_system_state: FileSystemState::default(),
         }
         .wrap_ok()
     }