Python: Debugging queries

python/lib/codeql-pack.lock.yml

@@ -2,25 +2,25 @@
 lockVersion: 1.0.0
 dependencies:
   codeql/dataflow:
-    version: 1.1.8
+    version: 2.0.4
   codeql/mad:
-    version: 1.0.14
+    version: 1.0.20
   codeql/python-all:
-    version: 3.1.0
+    version: 4.0.4
   codeql/regex:
-    version: 1.0.14
+    version: 1.0.20
   codeql/ssa:
-    version: 1.0.14
+    version: 1.0.20
   codeql/threat-models:
-    version: 1.0.14
+    version: 1.0.20
   codeql/tutorial:
-    version: 1.0.14
+    version: 1.0.20
   codeql/typetracking:
-    version: 1.0.14
+    version: 2.0.4
   codeql/util:
-    version: 2.0.1
+    version: 2.0.7
   codeql/xml:
-    version: 1.0.14
+    version: 1.0.20
   codeql/yaml:
-    version: 1.0.14
+    version: 1.0.20
 compiled: false

python/lib/ghsl.qll

@@ -0,0 +1,3 @@
+// Utils
+import ghsl.Utils
+import ghsl.Sinks

python/lib/ghsl/Sinks.qll

@@ -0,0 +1,72 @@
+private import python
+private import semmle.python.ApiGraphs
+private import semmle.python.Concepts
+private import semmle.python.dataflow.new.DataFlow
+
+private import semmle.python.security.dataflow.SqlInjectionCustomizations
+private import semmle.python.security.dataflow.CodeInjectionCustomizations
+private import semmle.python.security.dataflow.CommandInjectionCustomizations
+private import semmle.python.security.dataflow.LdapInjectionCustomizations
+private import semmle.python.security.dataflow.NoSqlInjectionCustomizations
+private import semmle.python.security.dataflow.ReflectedXSSCustomizations
+private import semmle.python.security.dataflow.UnsafeDeserializationCustomizations
+private import semmle.python.security.dataflow.XpathInjectionCustomizations
+private import semmle.python.security.dataflow.XxeCustomizations
+// Fields Sinks
+private import ghsl.HardcodedSecretSinks
+private import ghsl.MassAssignment
+
+
+/**
+ * List of all the sinks that we want to check.
+ */
+class AllSinks extends DataFlow::Node {
+  private string sink;
+
+  AllSinks() {
+    this instanceof MassAssignment::Sinks and
+    sink = "mass-assignment"
+    or
+    this instanceof CredentialSink and
+    sink = "credential"
+    or
+    this instanceof SqlInjection::Sink and
+    sink = "sql-injection"
+    or
+    this instanceof CodeInjection::Sink and
+    sink = "code-injection"
+    or
+    this instanceof CommandInjection::Sink and
+    sink = "command-injection"
+    or
+    (
+      this instanceof LdapInjection::DnSink
+      or
+      this instanceof LdapInjection::FilterSink
+    ) and
+    sink = "ldap-injection"
+    or
+    (
+      this instanceof NoSqlInjection::NoSqlExecutionAsDictSink and
+      this instanceof NoSqlInjection::NoSqlExecutionAsStringSink
+    ) and
+    sink = "nosql-injection"
+    or
+    this instanceof ReflectedXss::Sink and
+    sink = "reflected-xss"
+    or
+    this instanceof UnsafeDeserialization::Sink and
+    sink = "unsafe-deserialization"
+    or
+    this instanceof XpathInjection::Sink and
+    sink = "xpath-injection"
+    or
+    this instanceof Xxe::Sink and
+    sink = "xxe"
+  }
+
+  /**
+   * Gets the sink sink type.
+   */
+  string sinkType() { result = sink }
+}

python/lib/ghsl/Utils.qll

@@ -1,8 +1,61 @@
-import python
+private import python
 private import semmle.python.ApiGraphs
 private import semmle.python.Concepts
 private import semmle.python.dataflow.new.DataFlow
-private import semmle.python.dataflow.new.internal.TaintTrackingPrivate
+private import ghsl.LocalSources
+private import ghsl.Sinks
+
+/**
+ * Find Node at Location
+ */
+predicate filterByLocation(DataFlow::Node node, string relative_path, int linenumber) {
+  node.getLocation().getFile().getRelativePath() = relative_path and
+  node.getLocation().getStartLine() = linenumber
+}
+
+/**
+ * Check if the source node is a method parameter
+ */
+predicate functionParameters(DataFlow::Node node) {
+  (
+    // // Function Call Parameters
+    node instanceof DataFlow::ParameterNode
+    or
+    // Function Call Arguments
+    node instanceof DataFlow::ArgumentNode
+  ) and
+  node instanceof AllSinks and
+  node.getScope().inSource()
+}
+
+
+/**
+ * List of all the souces
+ */
+class AllSources extends DataFlow::Node {
+  private string threatmodel;
+
+  AllSources() {
+    exists(ThreatModelSource tms |
+      threatmodel = tms.getThreatModel() and
+      this = tms
+    )
+    or
+    this instanceof LocalSources::Range and
+    threatmodel = "local"
+  }
+
+  /**
+   * Gets the source threat model.
+   */
+  string getThreatModel() { result = threatmodel }
+}
+
+/**
+ * Local sources
+ */
+class LocalSources = LocalSources::Range;
+
 
 // List of all the format strings
 // - python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPrivate.qll
@@ -23,11 +76,11 @@ class DynamicStrings extends DataFlow::Node {
       exists(BinaryExpr expr |
         (
           // q = "WHERE name = %s" % username
-          expr.getOp() instanceof Mod or
+          expr.getOp() instanceof Mod
+          or
           // q = "WHERE name = " + username
           expr.getOp() instanceof Add
-        )
-        and
+        ) and
         expr.getLeft().getParent() = this.asExpr()
       )
     ) and

python/src/audit/templates/BackwardsPartialDataFlow.ql

@@ -13,7 +13,7 @@ import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
 import semmle.python.ApiGraphs
 import PartialFlow::PartialPathGraph
-import ghsl.Helpers // required for dangerousSinks
+import ghsl
 
 private module MyConfig implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { none() }

python/src/debugging/PartialPathsFromSink.ql

@@ -0,0 +1,44 @@
+/**
+ * @name Partial Path Query from Sink
+ * @kind path-problem
+ * @problem.severity warning
+ * @security-severity 1.0
+ * @sub-severity low
+ * @precision low
+ * @id py/debugging/partial-path-from-sink
+ * @tags debugging
+ */
+
+import python
+import ghsl
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.Concepts
+import semmle.python.dataflow.new.RemoteFlowSources
+import semmle.python.dataflow.new.BarrierGuards
+import semmle.python.ApiGraphs
+
+// Partial Graph
+module PartialFlowConfig implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node source) { any() }
+
+  predicate isSink(DataFlow::Node sink) { sink instanceof AllSinks }
+}
+
+int explorationLimit() { result = 10 }
+
+private module PartialFlows = DataFlow::Global<PartialFlowConfig>;
+
+private module PartialFlowsGraph = PartialFlows::FlowExplorationRev<explorationLimit/0>;
+
+private import PartialFlowsGraph::PartialPathGraph
+
+from PartialFlowsGraph::PartialPathNode source, PartialFlowsGraph::PartialPathNode sink
+where
+  /// Only show sinks from a certain file
+  // findByLocation(sink.getNode(), "File.java", _) and
+  /// Only show sources that match our criteria
+  // checkSource(source.getNode()) and
+  /// Partical Path
+  PartialFlowsGraph::partialFlow(source, sink, _)
+select sink.getNode(), source, sink, "Partial Graph $@.", source.getNode(), "user-provided value"

python/src/debugging/PartialPathsFromSource.ql

@@ -0,0 +1,47 @@
+/**
+ * @name Partial Path Query from Source
+ * @kind path-problem
+ * @problem.severity warning
+ * @security-severity 1.0
+ * @sub-severity low
+ * @precision low
+ * @id py/debugging/partial-path-from-source
+ * @tags debugging
+ */
+
+import python
+import ghsl
+import semmle.python.dataflow.new.DataFlow
+import semmle.python.dataflow.new.TaintTracking
+import semmle.python.Concepts
+import semmle.python.dataflow.new.RemoteFlowSources
+import semmle.python.dataflow.new.BarrierGuards
+import semmle.python.ApiGraphs
+
+// Partial Graph
+module PartialFlowConfig implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node source) {
+    source instanceof AllSources and
+    // Make sure the source node is in the source code
+    source.getScope().inSource()
+  }
+
+  predicate isSink(DataFlow::Node sink) { none() }
+}
+
+int explorationLimit() { result = 10 }
+
+module PartialFlows = DataFlow::Global<PartialFlowConfig>;
+
+module PartialFlowsGraph = PartialFlows::FlowExplorationFwd<explorationLimit/0>;
+
+import PartialFlowsGraph::PartialPathGraph
+
+from PartialFlowsGraph::PartialPathNode source, PartialFlowsGraph::PartialPathNode sink
+where
+  PartialFlowsGraph::partialFlow(source, sink, _) and
+  /// Filter by location
+  filterByLocation(source.getNode(), "app.py", _)
+/// Filter by Function Parameters
+// and functionParameters(sink.getNode())
+select sink.getNode(), source, sink, "Partial Graph $@.", source.getNode(), "user-provided value"

python/src/debugging/Sinks.ql

@@ -0,0 +1,16 @@
+/**
+ * @name List of all known sinks
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 1.0
+ * @sub-severity low
+ * @precision high
+ * @id py/debugging/sinks
+ * @tags debugging
+ */
+
+import python
+import ghsl
+
+from AllSinks sinks
+select sinks, "sink[" + sinks.sinkType() + "]"

python/src/debugging/Sources.ql

@@ -0,0 +1,19 @@
+/**
+ * @name List of all known sources (remote, local, etc.)
+ * @kind problem
+ * @problem.severity warning
+ * @security-severity 1.0
+ * @sub-severity low
+ * @precision high
+ * @id py/debugging/sources
+ * @tags debugging
+ */
+
+import python
+import ghsl
+
+from AllSources sources, string threatModel
+where threatModel = sources.getThreatModel()
+// Local sources
+// sources.getThreatModel() = "local"
+select sources, "source[" + threatModel + "]"

python/src/suites/python-debugging.qls

@@ -0,0 +1,19 @@
+- description: "GitHub's Community Packs Python Debugging Suite"
+
+- queries: '.'
+  from: githubsecuritylab/codeql-python-queries
+
+- include:
+    kind:
+    - problem
+    - path-problem
+    precision:
+    - very-high
+    - high
+    tags contain:
+      - debugging
+
+# Remove local testing folders
+- exclude:
+    query path:
+      - /testing\/.*/