Add external APIs query in extension

[koesie10]

This adds the external API query text to the extension directly to avoid users having to copy the query to their local codeql submodule or having to checkout a specific branch.

This is a temporary solution until the queries are stabilized. Once they are, we will upstream these to github/codeql and use them like other contextual queries.

Since this is just a temporary solution, this is not the prettiest code and is not intended to be a long-term solution. It does not do proper caching and will create a new temporary directory for every query run. The performance hit of this is acceptable and expected.

Checklist

• CHANGELOG.md has been updated to incorporate all user visible changes made by this pull request.
• Issues have been created for any UI or other user-facing changes made by this pull request.
• [Maintainers only] If this pull request makes user-facing changes that require documentation changes, open a corresponding docs pull request in the github/codeql repo and add the ready-for-doc-review label there.

[starcke]

Generally looks good to me, just a question about whether we can simplify.

[starcke]

I wonder if we should just inline ExternalApi.qll, so that we just have a single query file that we run instead of having to store multiple files?

[koesie10]

I think it would be easier to transfer these queries to the codeql repository if we leave them as separate files. So far, we've not had to make any modifications to the ExternalApi.qll file, so we would only need to add in the main query and not make any changes to the ExternalApi.qll file. If we do combine these files, we'd need to split them out again in the future.

[starcke]

I dont think it would be too hard to do that split up later, but I dont feel strongly. It also doesnt seem super complicated to have extra dependency files because we also need to have the synthetic pack. So happy to leave it like this.

[starcke]

Just a question, probably fine. Are languages just string or do we have a type for them?

[koesie10]

We do, but the DatabaseItem.language is a string, so we'd need to cast it to a QueryLanguage which might hide problems rather than actually giving us type safety. I've changed it to use a QueryLanguage though since that will prevent us from adding unsupported types.

[starcke]

If we want to go with just a single file the following should work:

/**
  * @name Usage of APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
  * @tags telemetry
  * @id xyz
  */
 
  private import java
  private import semmle.code.java.dataflow.DataFlow
  private import semmle.code.java.dataflow.ExternalFlow
  private import semmle.code.java.dataflow.FlowSources
  private import semmle.code.java.dataflow.FlowSummary
  private import semmle.code.java.dataflow.internal.DataFlowPrivate
  private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
  private import semmle.code.java.dataflow.TaintTracking
  
  pragma[nomagic]
  private predicate isTestPackage(Package p) {
    p.getName()
        .matches([
            "org.junit%", "junit.%", "org.mockito%", "org.assertj%",
            "com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
            "org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
            "org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
            "org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
            "org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
            "org.testng%"
          ])
  }
  
  /**
   * A test library.
   */
  private class TestLibrary extends RefType {
    TestLibrary() { isTestPackage(this.getPackage()) }
  }
  
  /** Holds if the given callable is not worth supporting. */
  private predicate isUninteresting(Callable c) {
    c.getDeclaringType() instanceof TestLibrary or
    c.(Constructor).isParameterless()
  }
  
  /**
   * An external API from either the Standard Library or a 3rd party library.
   */
  class ExternalApi extends Callable {
    ExternalApi() { not this.fromSource() and not isUninteresting(this) }
  
    /**
     * Gets information about the external API in the form expected by the MaD modeling framework.
     */
    string getApiName() {
      result =
        this.getDeclaringType().getPackage() + "." + this.getDeclaringType().getSourceDeclaration() +
          "#" + this.getName() + paramsString(this)
    }
  
  
    /** Gets a node that is an input to a call to this API. */
    private DataFlow::Node getAnInput() {
      exists(Call call | call.getCallee().getSourceDeclaration() = this |
        result.asExpr().(Argument).getCall() = call or
        result.(ArgumentNode).getCall().asCall() = call
      )
    }
  
    /** Gets a node that is an output from a call to this API. */
    private DataFlow::Node getAnOutput() {
      exists(Call call | call.getCallee().getSourceDeclaration() = this |
        result.asExpr() = call or
        result.(DataFlow::PostUpdateNode).getPreUpdateNode().(ArgumentNode).getCall().asCall() = call
      )
    }
  
    /** Holds if this API has a supported summary. */
    pragma[nomagic]
    predicate hasSummary() {
      this = any(SummarizedCallable sc).asCallable() or
      TaintTracking::localAdditionalTaintStep(this.getAnInput(), _)
    }
  
    pragma[nomagic]
    predicate isSource() {
      this.getAnOutput() instanceof RemoteFlowSource or sourceNode(this.getAnOutput(), _)
    }
  
    /** Holds if this API is a known sink. */
    pragma[nomagic]
    predicate isSink() { sinkNode(this.getAnInput(), _) }
  
    /** Holds if this API is a known neutral. */
    pragma[nomagic]
    predicate isNeutral() { this = any(FlowSummaryImpl::Public::NeutralCallable nsc).asCallable() }
  
    /**
     * Holds if this API is supported by existing CodeQL libraries, that is, it is either a
     * recognized source, sink or neutral or it has a flow summary.
     */
    predicate isSupported() {
      this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
    }
  }
  
 private Call aUsage(ExternalApi api) {
    result.getCallee().getSourceDeclaration() = api and
    not result.getFile() instanceof GeneratedFile
  }
  
  private boolean isSupported(ExternalApi api) {
   api.isSupported() and result = true
   or
   not api.isSupported() and
   result = false
  }
  
  from ExternalApi api, string apiName, boolean supported, Call usage
  where
    apiName = api.getApiName() and
    supported = isSupported(api) and
    usage = aUsage(api)
  select apiName, supported, usage

[starcke]

👍