Merge branch 'fix-large-sarif-handling' of github.com:marcnjaramillo/vscode-codeql into fix-large-sarif-handling

Author: marcnjaramillo

.github/workflows/main.yml

@@ -74,7 +74,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: [stable, nightly]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -104,31 +103,16 @@ jobs:
         run: |
           npm run lint
 
-      - name: Install CodeQL
-        run: |
-          mkdir codeql-home
-          if [ ${{ matrix.version }} = "stable" ]
-          then
-            curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
-          else
-            curl -L --silent ${{ needs.find-nightly.outputs.url }}/codeql.zip -o codeql-home/codeql.zip
-          fi
-          unzip -q -o codeql-home/codeql.zip -d codeql-home
-          unzip -q -o codeql-home/codeql.zip codeql/codeql.exe -d codeql-home
-          rm codeql-home/codeql.zip
-        shell: bash
-
       - name: Run unit tests (Linux)
         working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
         run: |
-          CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
+          npm run test
 
       - name: Run unit tests (Windows)
         if: matrix.os == 'windows-latest'
         working-directory: extensions/ql-vscode
         run: |
-          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
@@ -151,7 +135,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'nightly']
+        version: ['v2.3.3', 'v2.4.6', 'v2.5.9', 'v2.6.3', 'v2.7.0', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
       NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}

.vscode/extensions.json

@@ -3,7 +3,7 @@
   // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
   // List of extensions which should be recommended for users of this workspace.
   "recommendations": [
-    "eamodio.tsl-problem-matcher",
+    "amodio.tsl-problem-matcher",
     "dbaeumer.vscode-eslint",
     "eternalphane.tsfmt-vscode"
   ],

.vscode/launch.json

@@ -21,6 +21,9 @@
         // change to 'true' debug the IDE or Query servers
         "IDE_SERVER_JAVA_DEBUG": "false",
         "QUERY_SERVER_JAVA_DEBUG": "false",
+        "CLI_SERVER_JAVA_DEBUG": "false",
+        // Uncomment to set the JAVA_HOME for the codeql instance to use
+        // "CODEQL_JAVA_HOME": "/Library/Java/JavaVirtualMachines/jdk-12.0.1.jdk/Contents/Home"
       }
     },
     {
@@ -93,6 +96,19 @@
         // available in the workspace for the tests.
         // "${workspaceRoot}/../codeql"
       ],
+      "env": {
+        // Optionally, set the version to use for the integration tests.
+        // Use "nightly" to use the latest nightly build.
+        // "CLI_VERSION": "2.7.0",
+
+        // If CLI_VERSION is set to nightly, set this to the url of the nightly build.
+        // "NIGHTLY_URL": "some url to grab the nightly build",
+
+        // Optionally, add a path to the codeql executable to be used during these tests.
+        // If not specified, one will be downloaded automatically.
+        // This option overrides the CLI_VERSION option.
+        // "CLI_PATH": "${workspaceRoot}/../semmle-code/target/intree/codeql/codeql",
+      },
       "stopOnEntry": false,
       "sourceMaps": true,
       "outFiles": [

CONTRIBUTING.md

@@ -77,9 +77,9 @@ $ vscode/scripts/code-cli.sh --install-extension dist/vscode-codeql-*.vsix # if
 
 You can use VS Code to debug the extension without explicitly installing it. Just open this directory as a workspace in VS Code, and hit `F5` to start a debugging session.
 
-### Running the unit/integration tests
+### Running the unit tests and integration tests that do not require a CLI instance
 
-Ensure the `CODEQL_PATH` environment variable is set to point to the `codeql` cli executable.
+Unit tests and many integration tests do not require a copy of the CodeQL CLI.
 
 Outside of vscode, run:
 
@@ -89,6 +89,16 @@ npm run test && npm run integration
 
 Alternatively, you can run the tests inside of vscode. There are several vscode launch configurations defined that run the unit and integration tests. They can all be found in the debug view.
 
+Only the _With CLI_ tests require a CLI instance to run. See below on how to do that.
+
+Running from a terminal, you _must_ set the `TEST_CODEQL_PATH` variable to point to a checkout of the `github/codeql` repository. The appropriate CLI version will be downloaded as part of the test.
+
+### Running the integration tests
+
+The _Launch Integration Tests - With CLI_ tests require a CLI instance in order to run. There are several environment variables you can use to configure this.
+
+From inside of VSCode, open the `launch.json` file and in the _Launch Integration Tests - With CLI_ uncomment and change the environment variables appropriate for your purpose.
+
 ## Releasing (write access required)
 
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.

extensions/ql-vscode/CHANGELOG.md

@@ -3,9 +3,12 @@
 ## [UNRELEASED]
 
 - Fix a bug that causes VSCode to crash when handling large SARIF files (>4GB) [#1004](https://github.com/github/vscode-codeql/pull/1004)
+- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979) 
 - Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976)
 - Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
 - Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`)  in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)
+- Allow case-insensitive project slugs for GitHub repositories when adding a CodeQL database from LGTM. [#978](https://github.com/github/vscode-codeql/pull/961)
+- Make "Open Referenced File" command accessible from the active editor menu. [#989](https://github.com/github/vscode-codeql/pull/989)
 
 ## 1.5.6 - 07 October 2021
 

extensions/ql-vscode/src/cli.ts

@@ -22,6 +22,7 @@ import { assertNever } from './pure/helpers-pure';
 import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
 import { CompilationMessage } from './pure/messages';
+import { dbSchemeToLanguage } from './helpers';
 
 /**
  * The version of the SARIF format that we are using.
@@ -166,6 +167,11 @@ export class CodeQLCliServer implements Disposable {
   /** Version of current cli, lazily computed by the `getVersion()` method */
   private _version: SemVer | undefined;
 
+  /** 
+   * The languages supported by the current version of the CLI, computed by `getSupportedLanguages()`.
+   */
+  private _supportedLanguages: string[] | undefined;
+
   /** Path to current codeQL executable, or undefined if not running yet. */
   codeQlPath: string | undefined;
 
@@ -188,12 +194,14 @@ export class CodeQLCliServer implements Disposable {
       this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
         this._version = undefined;
+        this._supportedLanguages = undefined;
       });
     }
     if (this.cliConfig.onDidChangeConfiguration) {
       this.cliConfig.onDidChangeConfiguration(() => {
         this.restartCliServer();
         this._version = undefined;
+        this._supportedLanguages = undefined;
       });
     }
   }
@@ -264,11 +272,16 @@ export class CodeQLCliServer implements Disposable {
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
     const codeQlPath = await this.getCodeQlPath();
+    const args = [];
+    if (shouldDebugCliServer()) {
+      args.push('-J=-agentlib:jdwp=transport=dt_socket,address=localhost:9012,server=n,suspend=y,quiet=y');
+    }
+
     return await spawnServer(
       codeQlPath,
       'CodeQL CLI Server',
       ['execute', 'cli-server'],
-      [],
+      args,
       this.logger,
       _data => { /**/ }
     );
@@ -653,7 +666,7 @@ export class CodeQLCliServer implements Disposable {
     if (target) subcommandArgs.push('--target', target);
     if (name) subcommandArgs.push('--name', name);
     subcommandArgs.push(archivePath);
- 
+
     return await this.runCodeQlCliCommand(['database', 'unbundle'], subcommandArgs, `Extracting ${archivePath} to directory ${target}`);
   }
 
@@ -809,6 +822,23 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<LanguagesInfo>(['resolve', 'languages'], [], 'Resolving languages');
   }
 
+  /**
+   * Gets the list of available languages. Refines the result of `resolveLanguages()`, by excluding
+   * extra things like "xml" and "properties".
+   * 
+   * @returns An array of languages that are supported by the current version of the CodeQL CLI.
+   */
+  public async getSupportedLanguages(): Promise<string[]> {
+    if (!this._supportedLanguages) {
+      // Get the intersection of resolveLanguages with the list of hardcoded languages in dbSchemeToLanguage.
+      const resolvedLanguages = Object.keys(await this.resolveLanguages());
+      const hardcodedLanguages = Object.values(dbSchemeToLanguage);
+
+      this._supportedLanguages = resolvedLanguages.filter(lang => hardcodedLanguages.includes(lang));
+    }
+    return this._supportedLanguages;
+  }
+
   /**
    * Gets information about queries in a query suite.
    * @param suite The suite to resolve.
@@ -834,6 +864,39 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  async packInstall(dir: string) {
+    return this.runJsonCodeQlCliCommand(['pack', 'install'], [dir], 'Installing pack dependencies');
+  }
+
+  async packBundle(dir: string, workspaceFolders: string[], outputPath: string, precompile = true): Promise<void> {
+    const args = [
+      '-o',
+      outputPath,
+      dir,
+      '--additional-packs',
+      workspaceFolders.join(path.delimiter)
+    ];
+    if (!precompile && await this.cliConstraints.supportsNoPrecompile()) {
+      args.push('--no-precompile');
+    }
+
+    return this.runJsonCodeQlCliCommand(['pack', 'bundle'], args, 'Bundling pack');
+  }
+
+  async packPacklist(dir: string, includeQueries: boolean): Promise<string[]> {
+    const args = includeQueries ? [dir] : ['--no-include-queries', dir];
+    // since 2.7.1, packlist returns an object with a "paths" property that is a list of packs.
+    // previous versions return a list of packs.
+    const results: { paths: string[] } | string[] = await this.runJsonCodeQlCliCommand(['pack', 'packlist'], args, 'Generating the pack list');
+
+    // Once we no longer need to support 2.7.0 or earlier, we can remove this and assume all versions return an object.
+    if ('paths' in results) {
+      return results.paths;
+    } else {
+      return results;
+    }
+  }
+
   async generateDil(qloFile: string, outFile: string): Promise<void> {
     const extraArgs = await this.cliConstraints.supportsDecompileDil()
       ? ['--kind', 'dil', '-o', outFile, qloFile]
@@ -1082,6 +1145,12 @@ export function shouldDebugQueryServer() {
     && process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
 }
 
+export function shouldDebugCliServer() {
+  return 'CLI_SERVER_JAVA_DEBUG' in process.env
+    && process.env.CLI_SERVER_JAVA_DEBUG !== '0'
+    && process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== 'false';
+}
+
 export class CliVersionConstraint {
 
   /**
@@ -1121,6 +1190,16 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_DATABASE_UNBUNDLE = new SemVer('2.6.0');
 
+  /**
+   * CLI version where the `--no-precompile` option for pack creation was introduced.
+   */
+  public static CLI_VERSION_WITH_NO_PRECOMPILE = new SemVer('2.7.1');
+
+  /**
+   * CLI version where remote queries are supported.
+   */
+  public static CLI_VERSION_REMOTE_QUERIES = new SemVer('2.6.3');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1157,4 +1236,12 @@ export class CliVersionConstraint {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DATABASE_UNBUNDLE);
   }
 
+  async supportsNoPrecompile() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_NO_PRECOMPILE);
+  }
+
+  async supportsRemoteQueries() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_REMOTE_QUERIES);
+  }
+
 }

extensions/ql-vscode/src/config.ts

@@ -304,7 +304,7 @@ const REMOTE_QUERIES_SETTING = new Setting('remoteQueries', ROOT_SETTING);
 /**
  * Lists of GitHub repositories that you want to query remotely via the "Run Remote query" command.
  * Note: This command is only available for internal users.
- * 
+ *
  * This setting should be a JSON object where each key is a user-specified name (string),
  * and the value is an array of GitHub repositories (of the form `<owner>/<repo>`).
  */
@@ -314,6 +314,10 @@ export function getRemoteRepositoryLists(): Record<string, string[]> | undefined
   return REMOTE_REPO_LISTS.getValue<Record<string, string[]>>() || undefined;
 }
 
+export async function setRemoteRepositoryLists(lists: Record<string, string[]> | undefined) {
+  await REMOTE_REPO_LISTS.updateValue(lists, ConfigurationTarget.Global);
+}
+
 /**
  * The name of the "controller" repository that you want to use with the "Run Remote query" command.
  * Note: This command is only available for internal users.