add unit tests for storage

Signed-off-by: Piotr Dębski <piotrdebski@Piotrs-MacBook-Pro.local>

Author: Piotr Dębski

datafusion/vcf/examples/noodles_test.rs

@@ -1,27 +1,27 @@
-use std::io;
-use std::slice::SliceIndex;
-use std::sync::Arc;
 use bytes::Bytes;
 use datafusion::arrow::datatypes::DataType::Int64;
-use tokio_util::io::{StreamReader, SyncIoBridge};
-use noodles_bgzf as bgzf;
-use object_store::gcp::GoogleCloudStorageBuilder;
-use object_store::ObjectStore;
-use object_store::path::Path;
 use datafusion::error::Result;
 use datafusion::parquet::arrow::async_reader::AsyncFileReader;
 use futures::{FutureExt, StreamExt, TryStreamExt};
+use noodles_bgzf as bgzf;
+use object_store::ObjectStore;
+use object_store::gcp::GoogleCloudStorageBuilder;
+use object_store::path::Path;
+use std::io;
+use std::slice::SliceIndex;
+use std::sync::Arc;
+use tokio_util::io::{StreamReader, SyncIoBridge};
 // use noodles_vcf as vcf;
 use object_store::buffered::BufReader;
 use tokio::io::{AsyncBufRead, AsyncReadExt, AsyncSeek};
 
 // use opendal::Result;
+use log::{info, log};
+use noodles::{bam, sam, vcf};
+use opendal::Operator;
 use opendal::layers::LoggingLayer;
 use opendal::services;
-use opendal::Operator;
 use opendal::services::Gcs;
-use noodles::{bam, sam, vcf};
-use log::{info, log};
 use opendal::services::S3;
 
 // const BUCKET: &str = "gcp-public-data--gnomad";
@@ -40,18 +40,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .layer(LoggingLayer::default())
         .finish();
 
-    let stream = operator.reader_with(NAME)
+    let stream = operator
+        .reader_with(NAME)
         .chunk(16 * 1024 * 1024)
         .concurrent(2)
-        .await?.into_bytes_stream(..).await?;
+        .await?
+        .into_bytes_stream(..)
+        .await?;
     // let stream = operator.reader_with(NAME)
     //     .concurrent(8)
     //     .await?.into_bytes_stream(..).await?;
     let inner = bgzf::r#async::Reader::new(StreamReader::new(stream));
     let mut reader = vcf::r#async::io::Reader::new(inner);
     //
     info!("Reading header");
-    let mut  count = 0;
+    let mut count = 0;
     loop {
         let record = reader.records().next().await;
         if record.is_none() {
@@ -66,20 +69,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
-
-let builder = Gcs::default()
-    .bucket(BUCKET)
-    .disable_vm_metadata()
-    .allow_anonymous();
+// let builder = Gcs::default()
+//     .bucket(BUCKET)
+//     .disable_vm_metadata()
+//     .allow_anonymous();
 //
 // let operator = Operator::new(builder)?.finish();
 //
 // let stream = operator.reader(NAME).await?.into_bytes_stream(..).await?;
 // let inner = StreamReader::new(stream);
 
-
-
-
 // let stdout = io::stdout();
 // let mut writer = sam::r#async::io::Writer::new(stdout);
 //
@@ -124,4 +123,4 @@ let builder = Gcs::default()
 // header_reader.read_to_string(&mut raw_header).await?;
 // println!("{:}", raw_header);
 // let mut reader = bgzf::MultithreadedReader::new(stream);
-// let _inner = reader.get_mut();
+// let _inner = reader.get_mut();