Optimize write artifacts to disk in watch mode

crates/artifact_content/src/lib.rs

@@ -1,16 +1,18 @@
 mod eager_reader_artifact;
 mod entrypoint_artifact;
+mod file_system_state;
 mod format_parameter_type;
 pub mod generate_artifacts;
 mod imperatively_loaded_fields;
 mod import_statements;
 mod iso_overload_file;
 mod normalization_ast_text;
-mod operation_text;
+pub mod operation_text;
 mod persisted_documents;
 mod raw_response_type;
 mod reader_ast;
 mod refetch_reader_artifact;
 mod ts_config;
 
+pub use file_system_state::FileSystemState;
 pub use generate_artifacts::get_artifact_path_and_content;

crates/artifact_content/src/operation_text.rs

@@ -68,7 +68,7 @@ pub(crate) fn generate_operation_text<'a, TNetworkProtocol: NetworkProtocol>(
     }
 }
 
-fn hash(data: &str, algorithm: PersistedDocumentsHashAlgorithm) -> String {
+pub fn hash(data: &str, algorithm: PersistedDocumentsHashAlgorithm) -> String {
     match algorithm {
         PersistedDocumentsHashAlgorithm::Md5 => {
             let mut md5 = Md5::new();

crates/common_lang_types/src/file_system_operation.rs

@@ -0,0 +1,11 @@
+use std::path::PathBuf;
+
+use crate::FileContent;
+
+#[derive(Debug, Clone)]
+pub enum FileSystemOperation {
+    DeleteDirectory(PathBuf),
+    CreateDirectory(PathBuf),
+    WriteFile(PathBuf, FileContent),
+    DeleteFile(PathBuf),
+}

crates/common_lang_types/src/lib.rs

@@ -1,6 +1,7 @@
 mod absolute_and_relative_path;
 mod diagnostic;
 mod entity_and_selectable_name;
+mod file_system_operation;
 mod location;
 mod path_and_content;
 mod selectable_name;
@@ -12,6 +13,7 @@ mod text_with_carats;
 pub use absolute_and_relative_path::*;
 pub use diagnostic::*;
 pub use entity_and_selectable_name::*;
+pub use file_system_operation::*;
 pub use location::*;
 pub use path_and_content::*;
 pub use selectable_name::*;

crates/common_lang_types/src/path_and_content.rs

@@ -1,5 +1,6 @@
 use crate::{ArtifactFileName, ParentObjectEntityNameAndSelectableName};
 
+#[derive(Debug, Clone)]
 pub struct FileContent(pub String);
 
 impl From<String> for FileContent {
@@ -31,3 +32,32 @@ pub struct ArtifactPath {
     pub type_and_field: Option<ParentObjectEntityNameAndSelectableName>,
     pub file_name: ArtifactFileName,
 }
+
+#[derive(Debug, Clone)]
+pub struct ArtifactHash(String);
+
+impl From<String> for ArtifactHash {
+    fn from(value: String) -> Self {
+        ArtifactHash(value)
+    }
+}
+
+impl std::fmt::Display for ArtifactHash {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl std::ops::Deref for ArtifactHash {
+    type Target = String;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl PartialEq for ArtifactHash {
+    fn eq(&self, other: &Self) -> bool {
+        self.0 == other.0
+    }
+}

crates/isograph_compiler/src/batch_compile.rs

@@ -1,13 +1,14 @@
 use std::{path::PathBuf, time::Duration};
 
 use crate::{
-    compiler_state::CompilerState, with_duration::WithDuration,
-    write_artifacts::write_artifacts_to_disk,
+    compiler_state::CompilerState,
+    with_duration::WithDuration,
+    write_artifacts::{apply_file_system_operations, get_file_system_operations},
 };
 use artifact_content::get_artifact_path_and_content;
 use colored::Colorize;
 use common_lang_types::{CurrentWorkingDirectory, DiagnosticVecResult};
-use isograph_schema::{IsographDatabase, NetworkProtocol};
+use isograph_schema::NetworkProtocol;
 use prelude::Postfix;
 use pretty_duration::pretty_duration;
 use tracing::{error, info};
@@ -24,8 +25,10 @@ pub fn compile_and_print<TNetworkProtocol: NetworkProtocol>(
     current_working_directory: CurrentWorkingDirectory,
 ) -> DiagnosticVecResult<()> {
     info!("{}", "Starting to compile.".cyan());
-    let state = CompilerState::new(config_location, current_working_directory)?;
-    print_result(WithDuration::new(|| compile::<TNetworkProtocol>(&state.db)))
+    let mut state = CompilerState::new(config_location, current_working_directory)?;
+    print_result(WithDuration::new(|| {
+        compile::<TNetworkProtocol>(&mut state)
+    }))
 }
 
 pub fn print_result(
@@ -79,16 +82,22 @@ fn print_stats(elapsed_time: Duration, stats: CompilationStats) {
 /// This the "workhorse" command of batch compilation.
 #[tracing::instrument(skip_all)]
 pub fn compile<TNetworkProtocol: NetworkProtocol>(
-    db: &IsographDatabase<TNetworkProtocol>,
+    state: &mut CompilerState<TNetworkProtocol>,
 ) -> DiagnosticVecResult<CompilationStats> {
     // Note: we calculate all of the artifact paths and contents first, so that writing to
     // disk can be as fast as possible and we minimize the chance that changes to the file
     // system occur while we're writing and we get unpredictable results.
-
+    let db = &state.db;
     let config = db.get_isograph_config();
     let (artifacts, stats) = get_artifact_path_and_content(db)?;
-    let total_artifacts_written =
-        write_artifacts_to_disk(artifacts, &config.artifact_directory.absolute_path)?;
+
+    let file_system_operations = get_file_system_operations(
+        artifacts,
+        &config.artifact_directory.absolute_path,
+        &mut state.file_system_state,
+    );
+
+    let total_artifacts_written = apply_file_system_operations(file_system_operations)?;
 
     CompilationStats {
         client_field_count: stats.client_field_count,

crates/isograph_compiler/src/compiler_state.rs

@@ -10,13 +10,15 @@ use pico::Database;
 use prelude::Postfix;
 
 use crate::source_files::initialize_sources;
+use artifact_content::FileSystemState;
 
 const GC_DURATION_SECONDS: u64 = 60;
 
 #[derive(Debug)]
 pub struct CompilerState<TNetworkProtocol: NetworkProtocol> {
     pub db: IsographDatabase<TNetworkProtocol>,
     pub last_gc_run: Instant,
+    pub file_system_state: FileSystemState,
 }
 
 impl<TNetworkProtocol: NetworkProtocol> CompilerState<TNetworkProtocol> {
@@ -31,6 +33,7 @@ impl<TNetworkProtocol: NetworkProtocol> CompilerState<TNetworkProtocol> {
         Self {
             db,
             last_gc_run: Instant::now(),
+            file_system_state: FileSystemState::default(),
         }
         .wrap_ok()
     }

crates/isograph_compiler/src/watch.rs

@@ -30,7 +30,9 @@ pub async fn handle_watch_command<TNetworkProtocol: NetworkProtocol>(
     let config = state.db.get_isograph_config().clone();
 
     info!("{}", "Starting to compile.".cyan());
-    let _ = print_result(WithDuration::new(|| compile::<TNetworkProtocol>(&state.db)));
+    let _ = print_result(WithDuration::new(|| {
+        compile::<TNetworkProtocol>(&mut state)
+    }));
 
     let (mut file_system_receiver, mut file_system_watcher) =
         create_debounced_file_watcher(&config);
@@ -51,7 +53,7 @@ pub async fn handle_watch_command<TNetworkProtocol: NetworkProtocol>(
                     info!("{}", "File changes detected. Starting to compile.".cyan());
                     update_sources(&mut state.db, &changes)?;
                 };
-                let result = WithDuration::new(|| compile::<TNetworkProtocol>(&state.db));
+                let result = WithDuration::new(|| compile::<TNetworkProtocol>(&mut state));
                 let _ = print_result(result);
                 state.run_garbage_collection();
             }

crates/isograph_compiler/src/write_artifacts.rs

@@ -1,69 +1,71 @@
-use std::{
-    fs::{self, File},
-    io::Write,
-    path::PathBuf,
+use std::{fs, path::PathBuf};
+
+use common_lang_types::{
+    ArtifactPathAndContent, Diagnostic, DiagnosticResult, FileSystemOperation,
 };
 
-use common_lang_types::{ArtifactPathAndContent, Diagnostic, DiagnosticResult};
-use intern::string_key::Lookup;
+use artifact_content::FileSystemState;
 
 #[tracing::instrument(skip_all)]
-pub(crate) fn write_artifacts_to_disk(
+pub(crate) fn get_file_system_operations(
     paths_and_contents: impl IntoIterator<Item = ArtifactPathAndContent>,
     artifact_directory: &PathBuf,
-) -> DiagnosticResult<usize> {
-    if artifact_directory.exists() {
-        fs::remove_dir_all(artifact_directory).map_err(|e| {
-            unable_to_do_something_at_path_diagnostic(
-                artifact_directory,
-                &e.to_string(),
-                "delete directory",
-            )
-        })?;
-    }
-    fs::create_dir_all(artifact_directory).map_err(|e| {
-        let message = e.to_string();
-        unable_to_do_something_at_path_diagnostic(artifact_directory, &message, "create directory")
-    })?;
+    file_system_state: &mut FileSystemState,
+) -> Vec<FileSystemOperation> {
+    let artifacts: Vec<ArtifactPathAndContent> = paths_and_contents.into_iter().collect();
+    let new_file_system_state = FileSystemState::from_artifacts(artifacts);
+    let operations = file_system_state.diff(&new_file_system_state, artifact_directory);
+    *file_system_state = new_file_system_state;
+    operations
+}
 
+#[tracing::instrument(skip_all)]
+pub(crate) fn apply_file_system_operations(
+    operations: Vec<FileSystemOperation>,
+) -> DiagnosticResult<usize> {
     let mut count = 0;
-    for path_and_content in paths_and_contents {
-        // Is this better than materializing paths_and_contents sooner?
-        count += 1;
-
-        let absolute_directory = match path_and_content.artifact_path.type_and_field {
-            Some(type_and_field) => artifact_directory
-                .join(type_and_field.parent_object_entity_name.lookup())
-                .join(type_and_field.selectable_name.lookup()),
-            None => artifact_directory.clone(),
-        };
-        fs::create_dir_all(&absolute_directory).map_err(|e| {
-            unable_to_do_something_at_path_diagnostic(
-                &absolute_directory,
-                &e.to_string(),
-                "create directory",
-            )
-        })?;
 
-        let absolute_file_path =
-            absolute_directory.join(path_and_content.artifact_path.file_name.lookup());
-        let mut file = File::create(&absolute_file_path).map_err(|e| {
-            unable_to_do_something_at_path_diagnostic(
-                &absolute_file_path,
-                &e.to_string(),
-                "create file",
-            )
-        })?;
+    for operation in operations {
+        count += 1;
 
-        file.write(path_and_content.file_content.as_bytes())
-            .map_err(|e| {
-                unable_to_do_something_at_path_diagnostic(
-                    &absolute_file_path,
-                    &e.to_string(),
-                    "write contents of file",
-                )
-            })?;
+        match operation {
+            FileSystemOperation::DeleteDirectory(path) => {
+                if path.exists() {
+                    fs::remove_dir_all(path.clone()).map_err(|e| {
+                        unable_to_do_something_at_path_diagnostic(
+                            &path,
+                            &e.to_string(),
+                            "delete directory",
+                        )
+                    })?;
+                }
+            }
+            FileSystemOperation::CreateDirectory(path) => {
+                fs::create_dir_all(path.clone()).map_err(|e| {
+                    unable_to_do_something_at_path_diagnostic(
+                        &path,
+                        &e.to_string(),
+                        "create directory",
+                    )
+                })?;
+            }
+            FileSystemOperation::WriteFile(path, content) => {
+                fs::write(path.clone(), content.as_bytes()).map_err(|e| {
+                    unable_to_do_something_at_path_diagnostic(
+                        &path,
+                        &e.to_string(),
+                        "write contents of file",
+                    )
+                })?;
+            }
+            FileSystemOperation::DeleteFile(path) => {
+                fs::remove_file(path.clone()).map_err(|e| {
+                    unable_to_do_something_at_path_diagnostic(&path, &e.to_string(), "delete file")
+                })?;
+            }
+        }
     }
+
     Ok(count)
 }