add multi-parser support, AllJsonRecipes report

Summary:
With the addition of [JSON recipes in
Chef](chef/chef#15094), we'll need to make Bookworm
aware of other file types besides Ruby.

For this diff, I'm focusing mostly on adding JSON parsing (ie I'll be updating
the built-in rules+reports to handle JSON recipes in a subsequent diff).

Differential Revision: D79741933

fbshipit-source-id: f8b3d4a1aa6667ca6ddd13267112a25f304e3e1f

Author: dafyddcrosby

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/crawler.rb

@@ -13,12 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-require 'rubocop'
-require 'parser/current'
-
-# TODO(dcrosby) Bookworm key should determine which AST parser is chosen. This
-# would allow multiple AST parsers (ie ripper) and a way of ingesting non-Ruby
-# files like JSON/YAML
 
 module Bookworm
   class Crawler
@@ -47,23 +41,12 @@ def generate_file_queue(key)
     def process_paths(key)
       queue = @intake_queue[key]
       processed_files = {}
+      parser = BOOKWORM_KEYS[key]['parser']
       until queue.empty?
         path = queue.pop
-        processed_files[path] = generate_ast(File.read(path))
+        processed_files[path] = parser.parse(File.read(path))
       end
       @processed_files[key] = processed_files
     end
-
-    # In order to keep rules from barfing on a nil value (when no AST is
-    # generated at all from eg. an empty source code file), we supply a
-    # single node called bookworm_found_nil. It's a magic value that is
-    # 'unique enough' for our purposes
-    EMPTY_RUBOCOP_AST = ::RuboCop::AST::Node.new('bookworm_found_nil').freeze
-    def generate_rubocop_ast(code)
-      ::RuboCop::ProcessedSource.new(code, RUBY_VERSION.to_f)&.ast ||
-        EMPTY_RUBOCOP_AST
-    end
-
-    alias generate_ast generate_rubocop_ast
   end
 end

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/keys.rb

@@ -16,6 +16,8 @@
 # limitations under the License.
 #
 
+require 'bookworm/parsers/builtin_parsers'
+
 module Bookworm
   # These keys are how we generalize and generate a lot of code within Bookworm
   #
@@ -27,6 +29,9 @@ module Bookworm
   # dont_init_kb_key: Don't initialize the keys on knowledge base creation
   # determine_cookbook_name: Determines the cookbook name from the given path
   # path_name_regex: A regex with a capture to determine the prettified name of the file
+  #
+  # TODO: handle cookbook root alias files like cookbook/attributes.rb
+  # https://github.com/chef/chef/blob/d22fa4cdc46c58e450198427b86c4de4ed9a90e3/docs/dev/design_documents/cookbook_root_aliases.md?plain=1#L25-L28
   BOOKWORM_KEYS = {
     'cookbook' => {
       'metakey' => true,
@@ -46,6 +51,12 @@ module Bookworm
       'determine_cookbook_name' => true,
       'path_name_regex' => 'recipes/(.*)\.rb',
     },
+    'recipejson' => {
+      'determine_cookbook_name' => true,
+      'glob_pattern' => '*/recipes/*.json',
+      'path_name_regex' => 'recipes/(.*)\.json',
+      'parser' => ::Bookworm::Parsers::JSON,
+    },
     'attribute' => {
       'determine_cookbook_name' => true,
       'path_name_regex' => 'attributes/(.*)\.rb',
@@ -69,6 +80,7 @@ module Bookworm
   BOOKWORM_KEYS.each do |k, v|
     BOOKWORM_KEYS[k]['determine_cookbook_name'] ||= false
     BOOKWORM_KEYS[k]['plural'] ||= "#{k}s"
+    BOOKWORM_KEYS[k]['parser'] ||= ::Bookworm::Parsers::RuboCop
     BOOKWORM_KEYS[k]['source_dirs'] ||= 'cookbook_dirs'
     BOOKWORM_KEYS[k]['glob_pattern'] ||= "*/#{v['plural']}/*.rb"
   end

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/knowledge_base.rb

@@ -88,10 +88,11 @@ def create_pluralized_getter(key)
     def init_key(key, files)
       self[key] = {}
       path_name_regex = BOOKWORM_KEYS[key]['path_name_regex']
-      files.each do |path, ast|
+      parser_output_key = BOOKWORM_KEYS[key]['parser'].parser_output_key
+      files.each do |path, output|
         m = path.match(/#{path_name_regex}/)
         file_name = m[1]
-        self[key][file_name] = { 'path' => path, 'ast' => ast }
+        self[key][file_name] = { 'path' => path, parser_output_key => output }
       end
     end
 
@@ -104,13 +105,14 @@ def init_key_with_cookbook_name(key, files)
       self[key] = {}
       self['cookbook'] ||= {}
       path_name_regex = BOOKWORM_KEYS[key]['path_name_regex']
-      files.each do |path, ast|
+      parser_output_key = BOOKWORM_KEYS[key]['parser'].parser_output_key
+      files.each do |path, output|
         m = path.match(%r{/?([\w-]+)/#{path_name_regex}})
         cookbook_name = m[1]
         file_name = m[2]
         self['cookbook'][cookbook_name] ||= {}
         self[key]["#{cookbook_name}::#{file_name}"] =
-          { 'path' => path, 'cookbook' => cookbook_name, 'ast' => ast }
+          { 'path' => path, 'cookbook' => cookbook_name, parser_output_key => output }
       end
     end
   end

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/parser_base.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2025-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Bookworm
+  class KeyParserBase
+    def self.parse(str)
+      fail 'parse method required to inherit KeyParserBase class'
+    end
+  end
+end

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/parsers/builtin_parsers.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2025-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'bookworm/parser_base'
+
+require 'bookworm/parsers/rubocop'
+require 'bookworm/parsers/json'

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/parsers/json.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2025-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'json'
+
+module Bookworm
+  class Parsers
+    class JSON < ::Bookworm::KeyParserBase
+      def self.parse(str)
+        ::JSON.parse(str)
+      end
+
+      def self.parser_output_key
+        'object'
+      end
+    end
+  end
+end

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/parsers/rubocop.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2025-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'rubocop'
+require 'parser/current'
+
+module Bookworm
+  class Parsers
+    class RuboCop < ::Bookworm::KeyParserBase
+      # In order to keep rules from barfing on a nil value (when no AST is
+      # generated at all from eg. an empty source code file), we supply a
+      # single node called bookworm_found_nil. It's a magic value that is
+      # 'unique enough' for our purposes
+      EMPTY_RUBOCOP_AST = ::RuboCop::AST::Node.new('bookworm_found_nil').freeze
+      def self.parse(str)
+        ::RuboCop::ProcessedSource.new(str, RUBY_VERSION.to_f)&.ast ||
+          EMPTY_RUBOCOP_AST
+      end
+
+      def self.parser_output_key
+        'ast'
+      end
+    end
+  end
+
+  class Crawler
+    # TODO: Fix references to use Bookworm::Parsers::RuboCop::EMPTY_RUBOCOP_AST
+    EMPTY_RUBOCOP_AST = ::Bookworm::Parsers::RuboCop::EMPTY_RUBOCOP_AST
+  end
+end

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/reports/AllJsonRecipes.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2025-present, Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+description 'Lists all JSON Chef Recipes'
+needs_rules ['RecipeExists']
+
+def to_a
+  @kb.recipejsons.keys.sort
+end
+
+def output
+  to_a
+end

cookbooks/fb_bookworm/files/default/bookworm/lib/bookworm/rules/RecipeExists.rb

@@ -15,6 +15,7 @@
 description 'Helper rule to show existence of a recipe'
 keys %w{
   recipe
+  recipejson
 }
 
 def output