Merge branch 'aeisenberg/command-runner-refactor' into aeisenberg/telemetry

Author: aeisenberg

.github/workflows/main.yml

@@ -128,12 +128,11 @@ jobs:
         version: ['v2.2.6', 'v2.3.3', 'v2.4.0']
     env:
       CLI_VERSION: ${{ matrix.version }}
+      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
 
       - uses: actions/setup-node@v1
         with:
@@ -151,6 +150,12 @@ jobs:
           npm run build
         shell: bash
 
+      - name: Checkout QL
+        uses: actions/checkout@v2
+        with:
+          repository: github/codeql
+          path: codeql
+
       - name: Run CLI tests (Linux)
         working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'

.github/workflows/release.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd extensions/ql-vscode
-          npm install
+          npm ci
         shell: bash
 
       - name: Build
@@ -57,8 +57,6 @@ jobs:
           REF_NAME="$(echo ${{ github.ref }} | sed -e 's:^refs/tags/::' | sed -e 's:/:-:g')"
           echo "::set-output name=ref_name::$REF_NAME"
 
-      # Uploading artifacts is not necessary to create a release.
-      # This is just in case the release itself fails and we want to access the built artifacts from Actions.
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -139,3 +137,40 @@ jobs:
           body: This PR was automatically generated by the GitHub Actions release workflow in this repository.
           branch: ${{ format('version/bump-to-{0}', steps.bump-patch-version.outputs.next_version) }}
           base: main
+
+  vscode-publish:
+    name: Publish to VS Code Marketplace
+    needs: build
+    environment: publish-vscode-marketplace
+    runs-on: ubuntu-latest
+    env:
+      VSCE_TOKEN: ${{ secrets.VSCE_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx vsce publish -p $VSCE_TOKEN --packagePath *.vsix || \
+            echo "Failed to publish to VS Code Marketplace. \
+                  If this was an authentication problem, please make sure the \
+                  auth token hasn't expired."
+
+  open-vsx-publish:
+    name: Publish to Open VSX Registry
+    needs: build
+    environment: publish-open-vsx
+    runs-on: ubuntu-latest
+    env:
+      OPEN_VSX_TOKEN: ${{ secrets.OPEN_VSX_TOKEN }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: vscode-codeql-extension
+
+      - name: Publish to Registry
+        run: |
+          npx ovsx publish -p $OPEN_VSX_TOKEN *.vsix

.vscode/launch.json

@@ -8,7 +8,9 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode"
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        // Add a reference to a workspace to open. Eg-
+        // "${workspaceRoot}/../vscode-codeql-starter/vscode-codeql-starter.code-workspace"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -86,7 +88,7 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
-        "${workspaceRoot}/extensions/ql-vscode/test/data"
+        "${workspaceRoot}/extensions/ql-vscode/src/vscode-tests/cli-integration/data",
       ],
       "stopOnEntry": false,
       "sourceMaps": true,

CONTRIBUTING.md

@@ -114,14 +114,32 @@ Alternatively, you can run the tests inside of vscode. There are several vscode
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
-1. Log into the [Visual Studio Marketplace](https://marketplace.visualstudio.com/manage/publishers/github).
-1. Click the `...` menu in the CodeQL row and click **Update**.
-1. Drag the `.vsix` file you downloaded from the GitHub release into the Marketplace and click **Upload**.
-1. Go to the draft GitHub release, click 'Edit', add some summary description, and publish it.
+1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+    - If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
+1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
+1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
 1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
 1. If documentation changes need to be published, notify documentation team that release has been made.
 1. Review and merge the version bump PR that is automatically created by Actions.
 
+## Secrets and authentication for publishing
+
+Repository administrators, will need to manage the authentication keys for publishing to the VS Code marketplace and Open VSX. Each requires an authentication token. The VS Code marketplace token expires yearly.
+
+To regenerate the Open VSX token:
+
+1. Log in to the [user settings page on Open VSX](https://open-vsx.org/user-settings/namespaces).
+1. Make sure you are a member of the GitHub namespace.
+1. Go to the [Access Tokens](https://open-vsx.org/user-settings/tokens) page and generate a new token.
+1. Update the secret in the `publish-open-vsx` environment in the project settings.
+
+To regenerate the VSCode Marketplace token:
+
+1. Follow the instructions on [getting a PAT for Azure DevOps](https://code.visualstudio.com/api/working-with-extensions/publishing-extension#get-a-personal-access-token).
+1. Update the secret in the `publish-vscode-marketplace` environment in the project settings.
+
+Not that Azure DevOps PATs expire yearly and must be regenerated.
+
 ## Resources
 
 * [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)

extensions/ql-vscode/CHANGELOG.md

@@ -2,13 +2,19 @@
 
 ## [UNRELEASED]
 
-- Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting VS Code. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)
-- Fix bug when removing databases where sometimes the source folder would not be removed from the workspace or the database files would not be removed from the workspace storage location. [#692](https://github.com/github/vscode-codeql/pull/692)
+## 1.3.8 - 17 December 2020
+
+- Ensure databases are unlocked when removing them from the workspace. This will ensure that after a database is removed from VS Code, queries can be run on it from the command line without restarting the IDE. Requires CodeQL CLI 2.4.1 or later. [#681](https://github.com/github/vscode-codeql/pull/681)
+- Fix bug when removing databases where sometimes the source folder would not also be removed from the workspace or the database files would not be deleted from the workspace storage location. [#692](https://github.com/github/vscode-codeql/pull/692)
 - Query results with no string representation will now be displayed with placeholder text in query results. Previously, they were omitted. [#694](https://github.com/github/vscode-codeql/pull/694)
 - Add a label for the language of a database in the databases view. This will only take effect for new databases created with the CodeQL CLI v2.4.1 or later. [#697](https://github.com/github/vscode-codeql/pull/697)
 - Add clearer error message when running a query using a missing or invalid qlpack. [#702](https://github.com/github/vscode-codeql/pull/702)
 - Add clearer error message when trying to run a command from the query history view if no item in the history is selected. [#702](https://github.com/github/vscode-codeql/pull/702)
 - Fix a bug where it is not possible to download some database archives. This fix specifically addresses large archives and archives whose central directories do not align with file headers. [#700](https://github.com/github/vscode-codeql/pull/700)
+- Avoid error dialogs when QL test discovery or database cleanup encounters a missing directory. [#706](https://github.com/github/vscode-codeql/pull/706)
+- Add descriptive text and a link in the results view. [#711](https://github.com/github/vscode-codeql/pull/711)
+- Fix the _Set Label_ command in the query history view. [#710](https://github.com/github/vscode-codeql/pull/710)
+- Add the _CodeQL: View AST_ command to the right-click context menu when a source file in a database source archive is open in the editor. [#712](https://github.com/github/vscode-codeql/pull/712)
 
 ## 1.3.7 - 24 November 2020
 

extensions/ql-vscode/package-lock.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.3.8",
+  "version": "1.3.9",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -539,7 +539,8 @@
         },
         {
           "command": "codeQL.runQueries",
-          "group": "9_qlCommands"
+          "group": "9_qlCommands",
+          "when": "resourceScheme != codeql-zip-archive"
         }
       ],
       "commandPalette": [
@@ -669,6 +670,10 @@
           "command": "codeQL.runQuery",
           "when": "editorLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQL.viewAst",
+          "when": "resourceScheme == codeql-zip-archive"
+        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -755,7 +760,7 @@
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/fs-extra": "^9.0.3",
+    "@types/fs-extra": "^9.0.6",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",

extensions/ql-vscode/package.json

@@ -144,10 +144,15 @@ export class CodeQLCliServer implements Disposable {
   /** Path to current codeQL executable, or undefined if not running yet. */
   codeQlPath: string | undefined;
 
+  /**
+   * When set to true, ignore some modal popups and assume user has clicked "yes".
+   */
+  public quiet = false;
+
   constructor(
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    private logger: Logger
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;

extensions/ql-vscode/src/cli.ts

@@ -122,15 +122,16 @@ export function commandRunner(
       return await task(...args);
     } catch (e) {
       error = e;
+      const errorMessage = `${e.message || e} (${commandId})`;
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          logger.log(e.message);
+          logger.log(errorMessage);
         } else {
-          showAndLogWarningMessage(e.message);
+          showAndLogWarningMessage(errorMessage);
         }
       } else {
-        showAndLogErrorMessage(e.message || e);
+        showAndLogErrorMessage(errorMessage);
       }
       return undefined;
     } finally {
@@ -166,15 +167,16 @@ export function commandRunnerWithProgress<R>(
       return await withProgress(progressOptionsWithDefaults, task, ...args);
     } catch (e) {
       error = e;
+      const errorMessage = `${e.message || e} (${commandId})`;
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          logger.log(e.message);
+          logger.log(errorMessage);
         } else {
-          showAndLogWarningMessage(e.message);
+          showAndLogWarningMessage(errorMessage);
         }
       } else {
-        showAndLogErrorMessage(e.message || e);
+        showAndLogErrorMessage(errorMessage);
       }
       return undefined;
     } finally {
@@ -183,3 +185,45 @@ export function commandRunnerWithProgress<R>(
     }
   });
 }
+
+/**
+ * Displays a progress monitor that indicates how much progess has been made
+ * reading from a stream.
+ *
+ * @param readable The stream to read progress from
+ * @param messagePrefix A prefix for displaying the message
+ * @param totalNumBytes Total number of bytes in this stream
+ * @param progress The progress callback used to set messages
+ */
+export function reportStreamProgress(
+  readable: NodeJS.ReadableStream,
+  messagePrefix: string,
+  totalNumBytes?: number,
+  progress?: ProgressCallback
+) {
+  if (progress && totalNumBytes) {
+    let numBytesDownloaded = 0;
+    const bytesToDisplayMB = (numBytes: number): string => `${(numBytes / (1024 * 1024)).toFixed(1)} MB`;
+    const updateProgress = () => {
+      progress({
+        step: numBytesDownloaded,
+        maxStep: totalNumBytes,
+        message: `${messagePrefix} [${bytesToDisplayMB(numBytesDownloaded)} of ${bytesToDisplayMB(totalNumBytes)}]`,
+      });
+    };
+
+    // Display the progress straight away rather than waiting for the first chunk.
+    updateProgress();
+
+    readable.on('data', data => {
+      numBytesDownloaded += data.length;
+      updateProgress();
+    });
+  } else if (progress) {
+    progress({
+      step: 1,
+      maxStep: 2,
+      message: `${messagePrefix} (Size unknown)`,
+    });
+  }
+}

extensions/ql-vscode/src/commandRunner.ts

@@ -2,9 +2,9 @@ import * as vscode from 'vscode';
 
 import { decodeSourceArchiveUri, encodeArchiveBasePath, zipArchiveScheme } from '../archive-filesystem-provider';
 import { CodeQLCliServer } from '../cli';
-import { ProgressCallback, withProgress } from '../commandRunner';
 import { DatabaseManager } from '../databases';
 import { CachedOperation } from '../helpers';
+import { ProgressCallback, withProgress } from '../commandRunner';
 import * as messages from '../pure/messages';
 import { QueryServerClient } from '../queryserver-client';
 import { compileAndRunQueryAgainstDatabase, QueryWithResults } from '../run-queries';