Optimize write artifacts to disk in watch mode

crates/artifact_content/src/file_system_state.rs

@@ -0,0 +1,71 @@
+use std::collections::{BTreeSet, HashMap};
+
+use crate::operation_text::hash;
+use isograph_config::PersistedDocumentsHashAlgorithm;
+
+type Filepath = String;
+type FileContent = String;
+type FileHash = String;
+
+#[derive(Debug, Clone)]
+pub struct FileSystemState {
+    state: HashMap<Filepath, (FileContent, FileHash)>,
+    pub sorted_keys: BTreeSet<String>,
+}
+
+impl FileSystemState {
+    pub fn new() -> Self {
+        Self {
+            state: HashMap::new(),
+            sorted_keys: BTreeSet::new(),
+        }
+    }
+
+    pub fn insert(&mut self, filename: Filepath, content: FileContent) {
+        let hashed_content = hash(&content, PersistedDocumentsHashAlgorithm::Sha256);
+        self.sorted_keys.insert(filename.clone());
+        self.state.insert(filename, (content, hashed_content));
+    }
+
+    pub fn get_hashed_content(&self, filename: &str) -> Option<&FileHash> {
+        self.state
+            .get(filename)
+            .map(|(_, hashed_content)| hashed_content)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.state.is_empty()
+    }
+
+    fn difference(&self, other: &FileSystemState) -> Vec<String> {
+        self.sorted_keys
+            .difference(&other.sorted_keys)
+            .map(|k| k.clone())
+            .collect()
+    }
+
+    fn intersection(&self, other: &FileSystemState) -> Vec<String> {
+        self.sorted_keys
+            .intersection(&other.sorted_keys)
+            .map(|k| k.clone())
+            .collect()
+    }
+
+    pub fn compare(&self, other: &FileSystemState) -> (Vec<String>, Vec<String>) {
+        let to_delete = self.difference(other);
+        let mut to_add = other.difference(self);
+        let candidate_to_update = self.intersection(other);
+        for key in candidate_to_update {
+            if self.get_hashed_content(&key).unwrap() != other.get_hashed_content(&key).unwrap() {
+                to_add.push(key);
+            }
+        }
+        (to_delete, to_add)
+    }
+}
+
+impl Default for FileSystemState {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/artifact_content/src/lib.rs

@@ -1,16 +1,18 @@
 mod eager_reader_artifact;
 mod entrypoint_artifact;
+mod file_system_state;
 mod format_parameter_type;
 pub mod generate_artifacts;
 mod imperatively_loaded_fields;
 mod import_statements;
 mod iso_overload_file;
 mod normalization_ast_text;
-mod operation_text;
+pub mod operation_text;
 mod persisted_documents;
 mod raw_response_type;
 mod reader_ast;
 mod refetch_reader_artifact;
 mod ts_config;
 
+pub use file_system_state::FileSystemState;
 pub use generate_artifacts::get_artifact_path_and_content;

crates/artifact_content/src/operation_text.rs

@@ -68,7 +68,7 @@ pub(crate) fn generate_operation_text<'a, TNetworkProtocol: NetworkProtocol>(
     }
 }
 
-fn hash(data: &str, algorithm: PersistedDocumentsHashAlgorithm) -> String {
+pub fn hash(data: &str, algorithm: PersistedDocumentsHashAlgorithm) -> String {
     match algorithm {
         PersistedDocumentsHashAlgorithm::Md5 => {
             let mut md5 = Md5::new();

crates/isograph_compiler/src/batch_compile.rs

@@ -1,13 +1,14 @@
 use std::{path::PathBuf, time::Duration};
 
 use crate::{
-    compiler_state::CompilerState, with_duration::WithDuration,
-    write_artifacts::write_artifacts_to_disk,
+    compiler_state::CompilerState,
+    with_duration::WithDuration,
+    write_artifacts::{get_artifacts_to_write, write_artifacts_to_disk},
 };
 use artifact_content::get_artifact_path_and_content;
 use colored::Colorize;
 use common_lang_types::{CurrentWorkingDirectory, DiagnosticVecResult};
-use isograph_schema::{IsographDatabase, NetworkProtocol};
+use isograph_schema::NetworkProtocol;
 use prelude::Postfix;
 use pretty_duration::pretty_duration;
 use tracing::{error, info};
@@ -24,8 +25,10 @@ pub fn compile_and_print<TNetworkProtocol: NetworkProtocol>(
     current_working_directory: CurrentWorkingDirectory,
 ) -> DiagnosticVecResult<()> {
     info!("{}", "Starting to compile.".cyan());
-    let state = CompilerState::new(config_location, current_working_directory)?;
-    print_result(WithDuration::new(|| compile::<TNetworkProtocol>(&state.db)))
+    let mut state = CompilerState::new(config_location, current_working_directory)?;
+    print_result(WithDuration::new(|| {
+        compile::<TNetworkProtocol>(&mut state)
+    }))
 }
 
 pub fn print_result(
@@ -77,18 +80,25 @@ fn print_stats(elapsed_time: Duration, stats: CompilationStats) {
 }
 
 /// This the "workhorse" command of batch compilation.
-#[tracing::instrument(skip(db))]
+#[tracing::instrument(skip(state))]
 pub fn compile<TNetworkProtocol: NetworkProtocol>(
-    db: &IsographDatabase<TNetworkProtocol>,
+    state: &mut CompilerState<TNetworkProtocol>,
 ) -> DiagnosticVecResult<CompilationStats> {
     // Note: we calculate all of the artifact paths and contents first, so that writing to
     // disk can be as fast as possible and we minimize the chance that changes to the file
     // system occur while we're writing and we get unpredictable results.
-
+    let db = &state.db;
     let config = db.get_isograph_config();
     let (artifacts, stats) = get_artifact_path_and_content(db)?;
+
+    let artifacts_to_write = get_artifacts_to_write(
+        artifacts,
+        &config.artifact_directory.absolute_path,
+        &mut state.file_system_state,
+    );
+
     let total_artifacts_written =
-        write_artifacts_to_disk(artifacts, &config.artifact_directory.absolute_path)?;
+        write_artifacts_to_disk(artifacts_to_write, &config.artifact_directory.absolute_path)?;
 
     CompilationStats {
         client_field_count: stats.client_field_count,

crates/isograph_compiler/src/changed_artifacts.rs

@@ -0,0 +1,30 @@
+use std::{collections::HashMap, path::PathBuf};
+
+use common_lang_types::ArtifactPathAndContent;
+
+pub struct ChangedArtifacts {
+    pub artifacts_to_write: HashMap<PathBuf, ArtifactPathAndContent>,
+    pub artifacts_to_delete: Vec<PathBuf>,
+    pub cleanup_artifact_directory: bool,
+}
+
+impl ChangedArtifacts {
+    pub fn new() -> Self {
+        Self {
+            artifacts_to_write: HashMap::new(),
+            artifacts_to_delete: Vec::new(),
+            cleanup_artifact_directory: true,
+        }
+    }
+    pub fn delete(&mut self, paths: Vec<PathBuf>) {
+        for path in paths {
+            self.artifacts_to_delete.push(path.clone());
+        }
+    }
+}
+
+impl Default for ChangedArtifacts {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/isograph_compiler/src/compiler_state.rs

@@ -10,13 +10,15 @@ use pico::Database;
 use prelude::Postfix;
 
 use crate::source_files::initialize_sources;
+use artifact_content::FileSystemState;
 
 const GC_DURATION_SECONDS: u64 = 60;
 
 #[derive(Debug)]
 pub struct CompilerState<TNetworkProtocol: NetworkProtocol> {
     pub db: IsographDatabase<TNetworkProtocol>,
     pub last_gc_run: Instant,
+    pub file_system_state: FileSystemState,
 }
 
 impl<TNetworkProtocol: NetworkProtocol> CompilerState<TNetworkProtocol> {
@@ -31,6 +33,7 @@ impl<TNetworkProtocol: NetworkProtocol> CompilerState<TNetworkProtocol> {
         Self {
             db,
             last_gc_run: Instant::now(),
+            file_system_state: FileSystemState::new(),
         }
         .wrap_ok()
     }

crates/isograph_compiler/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod batch_compile;
+pub mod changed_artifacts;
 mod compiler_state;
 mod read_files;
 mod source_files;

crates/isograph_compiler/src/watch.rs

@@ -30,7 +30,9 @@ pub async fn handle_watch_command<TNetworkProtocol: NetworkProtocol>(
     let config = state.db.get_isograph_config().clone();
 
     info!("{}", "Starting to compile.".cyan());
-    let _ = print_result(WithDuration::new(|| compile::<TNetworkProtocol>(&state.db)));
+    let _ = print_result(WithDuration::new(|| {
+        compile::<TNetworkProtocol>(&mut state)
+    }));
 
     let (mut file_system_receiver, mut file_system_watcher) =
         create_debounced_file_watcher(&config);
@@ -51,7 +53,7 @@ pub async fn handle_watch_command<TNetworkProtocol: NetworkProtocol>(
                     info!("{}", "File changes detected. Starting to compile.".cyan());
                     update_sources(&mut state.db, &changes)?;
                 };
-                let result = WithDuration::new(|| compile::<TNetworkProtocol>(&state.db));
+                let result = WithDuration::new(|| compile::<TNetworkProtocol>(&mut state));
                 let _ = print_result(result);
                 state.run_garbage_collection();
             }