Add telemetry for commands

This commit adds telemetry capturing for command execution. The data
captured is only the command id. We also capture errors thrown by any
command execution.

There is a new config setting added that controls whether or not
telemetry should be sent. This setting AND the global setting must
be enabled in order for telemetry to be sent.

Note that the global setting is handled inside the extension by default.

Author: aeisenberg

extensions/ql-vscode/package-lock.json

@@ -169,6 +169,11 @@
           "minimum": 0,
           "maximum": 1024,
           "description": "Number of threads for running CodeQL tests."
+        },
+        "codeQL.telemetry.enableTelemetry": {
+          "type": "boolean",
+          "default": true,
+          "markdownDescription": "Specifies whether to enable Code QL telemetry. This setting AND the global `#telemetry.enableTelemetry#` setting must be checked for telemetry to be sent."
         }
       }
     },
@@ -710,6 +715,7 @@
     "tmp-promise": "~3.0.2",
     "tree-kill": "~1.2.2",
     "unzipper": "~0.10.5",
+    "vscode-extension-telemetry": "^0.1.6",
     "vscode-jsonrpc": "^5.0.1",
     "vscode-languageclient": "^6.1.3",
     "vscode-test-adapter-api": "~1.7.0",

extensions/ql-vscode/package.json

@@ -211,6 +211,8 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
   }
 }
 
+export const ENABLE_TELEMETRY = new Setting('telemetry.enableTelemetry', ROOT_SETTING);
+
 // Enable experimental features
 
 /**

extensions/ql-vscode/src/config.ts

@@ -58,6 +58,7 @@ import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
 import { CompareInterfaceManager } from './compare/compare-interface';
 import { gatherQlFiles } from './files';
+import { initializeTelemetry } from './telemetry';
 
 /**
  * extension.ts
@@ -86,6 +87,9 @@ const errorStubs: Disposable[] = [];
  */
 let isInstallingOrUpdatingDistribution = false;
 
+const extensionId = 'GitHub.vscode-codeql';
+const extension = extensions.getExtension(extensionId);
+
 /**
  * If the user tries to execute vscode commands after extension activation is failed, give
  * a sensible error message.
@@ -96,8 +100,6 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
   // Remove existing stubs
   errorStubs.forEach(stub => stub.dispose());
 
-  const extensionId = 'GitHub.vscode-codeql'; // TODO: Is there a better way of obtaining this?
-  const extension = extensions.getExtension(extensionId);
   if (extension === undefined) {
     throw new Error(`Can't find extension ${extensionId}`);
   }
@@ -113,9 +115,13 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
 }
 
 export async function activate(ctx: ExtensionContext): Promise<void> {
-  logger.log('Starting CodeQL extension');
+  logger.log(`Starting ${extensionId} extension`);
+  if (extension === undefined) {
+    throw new Error(`Can't find extension ${extensionId}`);
+  }
 
   initializeLogging(ctx);
+  initializeTelemetry(extension, ctx);
   languageSupport.install();
 
   const distributionConfigListener = new DistributionConfigListener();

extensions/ql-vscode/src/extension.ts

@@ -14,6 +14,7 @@ import {
 } from 'vscode';
 import { CodeQLCliServer } from './cli';
 import { logger } from './logging';
+import { sendCommandUsage } from './telemetry';
 
 export class UserCancellationException extends Error {
   /**
@@ -120,9 +121,13 @@ export function commandRunner(
   task: NoProgressTask,
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTIme = Date.now();
+    let error: Error | undefined;
+
     try {
       await task(...args);
     } catch (e) {
+      error = e;
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
@@ -133,6 +138,9 @@ export function commandRunner(
       } else {
         showAndLogErrorMessage(e.message || e);
       }
+    } finally {
+      const executionTime = Date.now() - startTIme;
+      sendCommandUsage(commandId, executionTime, error);
     }
   });
 }
@@ -153,13 +161,16 @@ export function commandRunnerWithProgress<R>(
   progressOptions: Partial<ProgressOptions>
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
+    const startTIme = Date.now();
+    let error: Error | undefined;
     const progressOptionsWithDefaults = {
       location: ProgressLocation.Notification,
       ...progressOptions
     };
     try {
       await withProgress(progressOptionsWithDefaults, task, ...args);
     } catch (e) {
+      error = e;
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
@@ -170,6 +181,9 @@ export function commandRunnerWithProgress<R>(
       } else {
         showAndLogErrorMessage(e.message || e);
       }
+    } finally {
+      const executionTime = Date.now() - startTIme;
+      sendCommandUsage(commandId, executionTime, error);
     }
   });
 }

extensions/ql-vscode/src/helpers.ts

@@ -0,0 +1,84 @@
+import { Extension, ExtensionContext, workspace} from 'vscode';
+import TelemetryReporter from 'vscode-extension-telemetry';
+import { Disposable } from 'vscode-jsonrpc';
+import { ENABLE_TELEMETRY } from './config';
+import { UserCancellationException } from './helpers';
+
+const key = '6f88c20e-2879-41ed-af73-218b82e1ff44';
+
+let reporter: TelemetryReporter | undefined;
+let listener: Disposable | undefined;
+
+export enum CommandCompletion {
+  Success = 'Success',
+  Failed = 'Failed',
+  Cancelled = 'Cancelled'
+}
+
+export function initializeTelemetry(extension: Extension<any>, ctx: ExtensionContext): void {
+  registerListener(extension, ctx);
+  if (reporter) {
+    reporter.dispose();
+    reporter = undefined;
+  }
+  if (ENABLE_TELEMETRY.getValue<boolean>()) {
+    reporter = new TelemetryReporter(extension.id, extension.packageJSON.version, key, /* anonymize stack traces */ true);
+    ctx.subscriptions.push(reporter);
+  }
+}
+
+export function sendCommandUsage(name: string, executionTime: number, error?: Error) {
+  if (!reporter) {
+    return;
+  }
+  const status = !error
+    ? CommandCompletion.Success
+    : error instanceof UserCancellationException
+    ? CommandCompletion.Cancelled
+    : CommandCompletion.Failed;
+
+  reporter.sendTelemetryEvent(
+    'command-usage',
+    {
+      name,
+      status,
+    },
+    { executionTime }
+  );
+
+  // if this is a true error, also report it
+  if (status === CommandCompletion.Failed) {
+    reporter.sendTelemetryException(
+      error!,
+      {
+        type: 'command-usage',
+        name,
+        status,
+      },
+      { executionTime }
+    );
+  }
+}
+
+function registerListener(extension: Extension<any>, ctx: ExtensionContext) {
+  if (!listener) {
+    listener = workspace.onDidChangeConfiguration(e => {
+      if (e.affectsConfiguration('codeQL.telemetry.enableTelemetry')) {
+        initializeTelemetry(extension, ctx);
+      }
+    });
+    ctx.subscriptions.push(listener);
+  }
+}
+
+// Exported for testing
+export function _dispose() {
+  if (listener) {
+    listener.dispose();
+    listener = undefined;
+  }
+  if (reporter) {
+    reporter.dispose();
+    reporter = undefined;
+  }
+}

extensions/ql-vscode/src/telemetry.ts

@@ -17,7 +17,8 @@ describe('databases', () => {
     databaseManager = new DatabaseManager(
       {
         workspaceState: {
-          update: updateSpy
+          update: updateSpy,
+          get: sinon.spy()
         }
       } as unknown as ExtensionContext,
       {} as QueryServerConfig,