[generator] Unstable APIs: add doc comments, filter out deprecated APIs, update main README

Author: swallez

Makefile.toml

@@ -169,7 +169,7 @@ description = "Generate Elasticsearch client documentation and opens in browser"
 clear = true
 category = "Elasticsearch"
 command = "cargo"
-args = ["doc", "-p", "elasticsearch", "--no-deps", "--open"]
+args = ["doc", "-p", "elasticsearch", "--no-deps", "--open", "--all-features"]
 
 [tasks.generate-release-notes]
 category = "Elasticsearch"

README.md

@@ -46,6 +46,11 @@ The following are a list of Cargo features that can be enabled or disabled:
 
 - **native-tls** *(enabled by default)*: Enables TLS functionality provided by `native-tls`.
 - **rustls-tls**: Enables TLS functionality provided by `rustls`.
+- **beta-apis**: Enables beta APIs. Beta APIs are on track to become stable and permanent features. Use them with 
+   caution because it is possible that breaking changes are made to these APIs in a minor version.
+- **experimental-apis**: Enables experimental APIs. Experimental APIs are just that - an experiment. An experimental
+  API might have breaking changes in any future version, or it might even be removed entirely. This feature also
+  enables `beta-apis`.
 
 ## Getting started
 

api_generator/Cargo.toml

@@ -16,7 +16,9 @@ flate2 = "~1"
 globset = "~0.4"
 Inflector = "0.11.4"
 indicatif = "0.12.0"
+itertools = "0.10.0"
 lazy_static = "1.4.0"
+log = "0.4.8"
 path-slash = "0.1.3"
 quote = "~0.3"
 reduce = "0.1.2"
@@ -26,6 +28,7 @@ semver = "0.9.0"
 serde = "~1"
 serde_json = "~1"
 serde_derive = "~1"
+simple_logger = "1.9.0"
 syn = { version = "~0.11", features = ["full"] }
 tar = "~0.4"
 toml = "0.5.6"

api_generator/src/bin/run.rs

@@ -28,6 +28,11 @@ use std::{
 };
 
 fn main() -> Result<(), failure::Error> {
+    simple_logger::SimpleLogger::new()
+        .with_level(log::LevelFilter::Info)
+        .init()
+        .unwrap();
+
     // This must be run from the repo root directory, with cargo make generate-api
     let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
     let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src"))?;

api_generator/src/generator/code_gen/mod.rs

@@ -22,7 +22,7 @@ pub mod request;
 pub mod root;
 pub mod url;
 
-use crate::generator::TypeKind;
+use crate::generator::{Stability, TypeKind};
 use inflector::Inflector;
 use quote::Tokens;
 use std::str;
@@ -72,6 +72,22 @@ fn doc<I: Into<String>>(comment: I) -> syn::Attribute {
     }
 }
 
+fn stability_doc(stability: Stability) -> Option<syn::Attribute> {
+    match stability {
+        Stability::Experimental => Some(doc(r#"&nbsp;
+# Optional, experimental
+This requires the `experimental-apis` feature. Can have breaking changes in future
+versions or might even be removed entirely.
+        "#)),
+        Stability::Beta => Some(doc(r#"&nbsp;
+# Optional, beta
+This requires the `beta-apis` feature. On track to become stable but breaking changes can
+happen in minor versions.
+        "#)),
+        Stability::Stable => None,
+    }
+}
+
 /// AST for an expression parsed from quoted tokens
 pub fn parse_expr(input: quote::Tokens) -> syn::Expr {
     syn::parse_expr(input.to_string().as_ref()).unwrap()

api_generator/src/generator/code_gen/namespace_clients.rs

@@ -33,6 +33,11 @@ pub fn generate(api: &Api, docs_dir: &PathBuf) -> Result<Vec<(String, String)>,
         if let Some(attr) = namespace.stability.inner_cfg_attr() {
             tokens.append(attr);
         }
+        if let Some(mut attr) = stability_doc(namespace.stability) {
+            attr.style = syn::AttrStyle::Inner;
+            tokens.append(quote! { #attr });
+        }
+
         tokens.append(use_declarations());
 
         let namespace_pascal_case = namespace_name.to_pascal_case();
@@ -77,10 +82,12 @@ pub fn generate(api: &Api, docs_dir: &PathBuf) -> Result<Vec<(String, String)>,
             .unzip();
 
         let cfg_attr = namespace.stability.outer_cfg_attr();
+        let cfg_doc = stability_doc(namespace.stability);
         tokens.append(quote!(
             #(#builders)*
 
             #namespace_doc
+            #cfg_doc
             #cfg_attr
             pub struct #namespace_client_name<'a> {
                 transport: &'a Transport

api_generator/src/generator/code_gen/params.rs

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+use crate::generator::code_gen::stability_doc;
 use crate::generator::*;
 use inflector::Inflector;
 use quote::Tokens;
@@ -65,11 +66,13 @@ fn generate_param(tokens: &mut Tokens, e: &ApiEnum) {
     };
 
     let cfg_attr = e.stability.outer_cfg_attr();
+    let cfg_doc = stability_doc(e.stability);
 
     let generated_enum_tokens = quote!(
+        #doc
+        #cfg_doc
         #cfg_attr
         #[derive(Debug, PartialEq, Deserialize, Serialize, Clone, Copy)]
-        #doc
         pub enum #name {
             #(#[serde(rename = #renames)] #variants),*
         }

api_generator/src/generator/code_gen/request/request_builder.rs

@@ -627,6 +627,7 @@ impl<'a> RequestBuilder<'a> {
         ));
 
         let cfg_attr = endpoint.stability.outer_cfg_attr();
+        let cfg_doc = stability_doc(endpoint.stability);
 
         quote! {
             #cfg_attr
@@ -635,9 +636,10 @@ impl<'a> RequestBuilder<'a> {
             #cfg_attr
             #enum_impl
 
-            #[derive(Clone, Debug)]
             #[doc = #builder_doc]
+            #cfg_doc
             #cfg_attr
+            #[derive(Clone, Debug)]
             pub struct #builder_expr {
                 transport: &'a Transport,
                 parts: #enum_ty,
@@ -676,6 +678,7 @@ impl<'a> RequestBuilder<'a> {
         enum_builder: &EnumBuilder,
     ) -> Tokens {
         let cfg_attr = endpoint.stability.outer_cfg_attr();
+        let cfg_doc = stability_doc(endpoint.stability);
 
         let builder_ident = ident(builder_name);
 
@@ -734,6 +737,7 @@ impl<'a> RequestBuilder<'a> {
         if enum_builder.contains_single_parameterless_part() {
             quote!(
                 #method_doc
+                #cfg_doc
                 #cfg_attr
                 pub fn #fn_name(&'a self) -> #builder_ident_ret {
                     #builder_ident::new(#clone_expr)
@@ -743,6 +747,7 @@ impl<'a> RequestBuilder<'a> {
             let (enum_ty, _, _) = enum_builder.clone().build();
             quote!(
                 #method_doc
+                #cfg_doc
                 #cfg_attr
                 pub fn #fn_name(&'a self, parts: #enum_ty) -> #builder_ident_ret {
                     #builder_ident::new(#clone_expr, parts)

api_generator/src/generator/code_gen/url/enum_builder.rs

@@ -327,6 +327,7 @@ mod tests {
                     url: None,
                 },
                 stability: Stability::Stable,
+                deprecated: None,
                 url: Url {
                     paths: vec![
                         Path {

api_generator/src/generator/mod.rs

@@ -42,7 +42,9 @@ use void::Void;
 pub mod code_gen;
 pub mod output;
 
+use itertools::Itertools;
 use output::{merge_file, write_file};
+use std::cmp::Ordering;
 
 lazy_static! {
     static ref VERSION: Version = semver::Version::parse(env!("CARGO_PKG_VERSION")).unwrap();
@@ -205,6 +207,37 @@ pub struct Deprecated {
     pub description: String,
 }
 
+impl PartialOrd for Deprecated {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        match (
+            Version::parse(&self.version),
+            Version::parse(&other.version),
+        ) {
+            (Err(_), _) => None,
+            (_, Err(_)) => None,
+            (Ok(self_version), Ok(other_version)) => self_version.partial_cmp(&other_version),
+        }
+    }
+}
+
+impl Deprecated {
+    /// Combine optional deprecations, keeping either lack of deprecation or the highest version
+    pub fn combine<'a>(
+        left: &'a Option<Deprecated>,
+        right: &'a Option<Deprecated>,
+    ) -> &'a Option<Deprecated> {
+        if let (Some(leftd), Some(rightd)) = (left, right) {
+            if leftd > rightd {
+                left
+            } else {
+                right
+            }
+        } else {
+            &None
+        }
+    }
+}
+
 /// An API url path
 #[derive(Debug, PartialEq, Deserialize, Clone)]
 pub struct Path {
@@ -392,6 +425,7 @@ pub struct ApiEndpoint {
     documentation: Documentation,
     pub stability: Stability,
     pub url: Url,
+    pub deprecated: Option<Deprecated>,
     #[serde(default = "BTreeMap::new")]
     pub params: BTreeMap<String, Type>,
     pub body: Option<Body>,
@@ -583,6 +617,11 @@ pub fn read_api(branch: &str, download_dir: &PathBuf) -> Result<Api, failure::Er
             let mut file = File::open(&path)?;
             let (name, api_endpoint) = endpoint_from_file(display, &mut file)?;
 
+            if api_endpoint.stability != Stability::Stable && api_endpoint.deprecated.is_some() {
+                // Do not generate deprecated unstable endpoints
+                continue;
+            }
+
             let name_parts: Vec<&str> = name.splitn(2, '.').collect();
             let (namespace, method_name) = match name_parts.len() {
                 len if len > 1 => (name_parts[0].to_string(), name_parts[1].to_string()),
@@ -655,21 +694,37 @@ where
     R: Read,
 {
     // deserialize the map from the reader
-    let endpoint: BTreeMap<String, ApiEndpoint> =
+    let endpoints: BTreeMap<String, ApiEndpoint> =
         serde_json::from_reader(reader).map_err(|e| super::error::ParseError {
             message: format!("Failed to parse {} because: {}", name, e),
         })?;
 
     // get the first (and only) endpoint name and endpoint body
-    let mut first_endpoint = endpoint.into_iter().next().unwrap();
-    first_endpoint.1.full_name = Some(first_endpoint.0.clone());
+    let (name, mut endpoint) = endpoints.into_iter().next().unwrap();
+    endpoint.full_name = Some(name.clone());
 
     // sort the HTTP methods so that we can easily pattern match on them later
-    for path in first_endpoint.1.url.paths.iter_mut() {
+    for path in endpoint.url.paths.iter_mut() {
         path.methods.sort();
     }
 
-    Ok(first_endpoint)
+    // endpoint deprecation is the "least deprecated" of its paths
+    let deprecation = endpoint
+        .url
+        .paths
+        .iter()
+        .map(|p| &p.deprecated)
+        .fold1(|d1, d2| Deprecated::combine(d1, d2))
+        .unwrap_or(&None);
+
+    if let Some(deprecated) = deprecation {
+        endpoint.deprecated = Some(Deprecated {
+            version: deprecated.version.clone(),
+            description: "Deprecated via one of the child items".to_string(),
+        })
+    }
+
+    Ok((name, endpoint))
 }
 
 /// deserializes Common from a file
@@ -699,4 +754,20 @@ mod tests {
         assert!(Stability::Beta > Stability::Experimental);
         assert!(Stability::Stable > Stability::Beta);
     }
+
+    #[test]
+    fn combine_deprecations() {
+        let d1 = Some(Deprecated {
+            version: "7.5.0".to_string(),
+            description: "foo".to_string(),
+        });
+
+        let d2 = Some(Deprecated {
+            version: "7.6.0".to_string(),
+            description: "foo".to_string(),
+        });
+
+        assert_eq!(&d2, Deprecated::combine(&d1, &d2));
+        assert_eq!(&None, Deprecated::combine(&d1, &None));
+    }
 }