From c5377a563d6eb9e9a817bd49a2a342adf8343f2f Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 2 Apr 2019 07:43:32 -0700 Subject: [PATCH 001/105] Starting from scratch for the next version rewrite --- .gitignore | 3 + karma/configs/base.js | 83 ---- karma/configs/chrome.js | 22 - karma/configs/firefox.js | 12 - karma/configs/headless.js | 29 -- karma/configs/ie.js | 12 - karma/fixtures/clarity.fixture.html | 17 - karma/setup/clarity.ts | 148 ------ karma/setup/mocks/error.ts | 8 - karma/setup/mocks/event.ts | 25 - karma/setup/mocks/performance.ts | 130 ----- karma/setup/page.ts | 43 -- karma/setup/proxyapis/eventlistener.ts | 85 ---- karma/setup/proxyapis/jasmineclock.ts | 24 - karma/setup/proxyapis/mutationobserver.ts | 24 - karma/setup/proxyapis/performance.ts | 18 - karma/setup/proxyapis/worker.ts | 70 --- karma/setup/pubsub.ts | 103 ---- karma/setup/testasync.ts | 9 - karma/setup/uncompress.ts | 190 ------- karma/setup/watch.ts | 91 ---- karma/tests/_setup/asyncstart.ts | 50 -- karma/tests/_setup/pubsub.ts | 21 - karma/tests/_setup/syncstart.ts | 17 - karma/tests/compressionworker.ts | 158 ------ karma/tests/convert.ts | 28 -- karma/tests/core/compression.ts | 18 - karma/tests/core/core.ts | 58 --- karma/tests/core/teardown.ts | 46 -- karma/tests/core/trigger.ts | 28 -- karma/tests/core/upload.ts | 155 ------ karma/tests/instrumentation/activate.ts | 69 --- karma/tests/instrumentation/error.ts | 101 ---- karma/tests/instrumentation/perf.ts | 63 --- karma/tests/instrumentation/trigger.ts | 29 -- karma/tests/layout/cssrules.ts | 35 -- karma/tests/layout/input.ts | 60 --- karma/tests/layout/mutation.ts | 547 --------------------- karma/tests/layout/privacy.ts | 94 ---- karma/tests/layout/scroll.ts | 102 ---- karma/tests/layout/unmask.ts | 81 --- karma/tests/performance.ts | 126 ----- karma/tests/pointer.ts | 125 ----- package.json | 6 +- src/clarity.ts | 22 +- src/compress.ts | 238 --------- src/compressionworker.ts | 110 ----- src/config.ts | 32 -- src/converters/convert.ts | 3 - src/converters/fromarray.ts | 67 --- src/converters/schema.md | 66 --- src/converters/schema.ts | 96 ---- src/converters/toarray.ts | 46 -- src/core.ts | 386 +-------------- src/index.ts | 3 +- src/plugins.ts | 19 - src/plugins/errors.ts | 38 -- src/plugins/layout/layout.ts | 344 ------------- src/plugins/layout/nodeinfo.ts | 24 - src/plugins/layout/shadowdom.ts | 412 ---------------- src/plugins/layout/states/doctype.ts | 12 - src/plugins/layout/states/element.ts | 149 ------ src/plugins/layout/states/generic.ts | 29 -- src/plugins/layout/states/ignore.ts | 41 -- src/plugins/layout/states/stateprovider.ts | 40 -- src/plugins/layout/states/style.ts | 51 -- src/plugins/layout/states/text.ts | 22 - src/plugins/performance.ts | 174 ------- src/plugins/pointer/mouse.ts | 22 - src/plugins/pointer/pointer.ts | 70 --- src/plugins/pointer/touch.ts | 31 -- src/plugins/viewport.ts | 85 ---- src/upload.ts | 157 ------ src/utils.ts | 116 ----- types/compressionworker.d.ts | 28 -- types/config.d.ts | 67 --- types/core.d.ts | 80 --- types/index.d.ts | 33 -- types/instrumentation.d.ts | 103 ---- types/layout.d.ts | 133 ----- types/performance.d.ts | 22 - types/pointer.d.ts | 25 - types/viewport.d.ts | 25 - 83 files changed, 8 insertions(+), 6576 deletions(-) delete mode 100644 karma/configs/base.js delete mode 100644 karma/configs/chrome.js delete mode 100644 karma/configs/firefox.js delete mode 100644 karma/configs/headless.js delete mode 100644 karma/configs/ie.js delete mode 100644 karma/fixtures/clarity.fixture.html delete mode 100644 karma/setup/clarity.ts delete mode 100644 karma/setup/mocks/error.ts delete mode 100644 karma/setup/mocks/event.ts delete mode 100644 karma/setup/mocks/performance.ts delete mode 100644 karma/setup/page.ts delete mode 100644 karma/setup/proxyapis/eventlistener.ts delete mode 100644 karma/setup/proxyapis/jasmineclock.ts delete mode 100644 karma/setup/proxyapis/mutationobserver.ts delete mode 100644 karma/setup/proxyapis/performance.ts delete mode 100644 karma/setup/proxyapis/worker.ts delete mode 100644 karma/setup/pubsub.ts delete mode 100644 karma/setup/testasync.ts delete mode 100644 karma/setup/uncompress.ts delete mode 100644 karma/setup/watch.ts delete mode 100644 karma/tests/_setup/asyncstart.ts delete mode 100644 karma/tests/_setup/pubsub.ts delete mode 100644 karma/tests/_setup/syncstart.ts delete mode 100644 karma/tests/compressionworker.ts delete mode 100644 karma/tests/convert.ts delete mode 100644 karma/tests/core/compression.ts delete mode 100644 karma/tests/core/core.ts delete mode 100644 karma/tests/core/teardown.ts delete mode 100644 karma/tests/core/trigger.ts delete mode 100644 karma/tests/core/upload.ts delete mode 100644 karma/tests/instrumentation/activate.ts delete mode 100644 karma/tests/instrumentation/error.ts delete mode 100644 karma/tests/instrumentation/perf.ts delete mode 100644 karma/tests/instrumentation/trigger.ts delete mode 100644 karma/tests/layout/cssrules.ts delete mode 100644 karma/tests/layout/input.ts delete mode 100644 karma/tests/layout/mutation.ts delete mode 100644 karma/tests/layout/privacy.ts delete mode 100644 karma/tests/layout/scroll.ts delete mode 100644 karma/tests/layout/unmask.ts delete mode 100644 karma/tests/performance.ts delete mode 100644 karma/tests/pointer.ts delete mode 100644 src/compress.ts delete mode 100644 src/compressionworker.ts delete mode 100644 src/config.ts delete mode 100644 src/converters/convert.ts delete mode 100644 src/converters/fromarray.ts delete mode 100644 src/converters/schema.md delete mode 100644 src/converters/schema.ts delete mode 100644 src/converters/toarray.ts delete mode 100644 src/plugins.ts delete mode 100644 src/plugins/errors.ts delete mode 100644 src/plugins/layout/layout.ts delete mode 100644 src/plugins/layout/nodeinfo.ts delete mode 100644 src/plugins/layout/shadowdom.ts delete mode 100644 src/plugins/layout/states/doctype.ts delete mode 100644 src/plugins/layout/states/element.ts delete mode 100644 src/plugins/layout/states/generic.ts delete mode 100644 src/plugins/layout/states/ignore.ts delete mode 100644 src/plugins/layout/states/stateprovider.ts delete mode 100644 src/plugins/layout/states/style.ts delete mode 100644 src/plugins/layout/states/text.ts delete mode 100644 src/plugins/performance.ts delete mode 100644 src/plugins/pointer/mouse.ts delete mode 100644 src/plugins/pointer/pointer.ts delete mode 100644 src/plugins/pointer/touch.ts delete mode 100644 src/plugins/viewport.ts delete mode 100644 src/upload.ts delete mode 100644 src/utils.ts delete mode 100644 types/compressionworker.d.ts delete mode 100644 types/config.d.ts delete mode 100644 types/instrumentation.d.ts delete mode 100644 types/layout.d.ts delete mode 100644 types/performance.d.ts delete mode 100644 types/pointer.d.ts delete mode 100644 types/viewport.d.ts diff --git a/.gitignore b/.gitignore index 9e8bee9d..16bc9692 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ # Build results build/ +# npm packages +.rpt2_cache + # npm packages node_modules diff --git a/karma/configs/base.js b/karma/configs/base.js deleted file mode 100644 index ca47f2c1..00000000 --- a/karma/configs/base.js +++ /dev/null @@ -1,83 +0,0 @@ -"use strict"; -module.exports = function (config) { - config.set({ - - // Base path that will be used to resolve all patterns (eg. files, exclude). - basePath: "../..", - - // Frameworks to use. - // Available frameworks: https://npmjs.org/browse/keyword/karma-adapter - frameworks: ["karma-typescript", "fixture", "jasmine"], - - // List of files to load in the browser. - files: [ - "karma/fixtures/**/*.html", - "package.json", - "build/clarity.min.js", - "src/converters/**/*.ts", - "karma/setup/**/*.ts", - "karma/tests/**/*.ts", - "src/**/*.ts" - ], - - // Preprocess matching files before serving them to the browser. - // Available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor - preprocessors: { - "**/*.ts": ["karma-typescript"], - "**/*.html": ["html2js"], - "**/*.json": ["json_fixtures"], - }, - - karmaTypescriptConfig: { - tsconfig: "tsconfig.json", - - compilerOptions: { - "sourceMap": true - }, - - bundlerOptions: { - // There is a module resolution bug for typings files with tsconfig paths - // A workaround is to add each such file to bundlerOptions.exclude - // https://github.com/monounity/karma-typescript/issues/315#issuecomment-461746455 - exclude: [ - "@clarity-types/compressionworker", - "@clarity-types/config", - "@clarity-types/core", - "@clarity-types/index", - "@clarity-types/instrumentation", - "@clarity-types/layout", - "@clarity-types/performance", - "@clarity-types/pointer", - "@clarity-types/viewport" - ] - }, - - // Exclude all files from coverage - // NOTE: When we want to run coverage, we need a way to exclude BLOB files created by compression workers - // Otherwise Karma can't map those blobs to an actual file and it causes an error '__cov_..... is undefined' - coverageOptions: { - exclude: [/^.*$/] - } - }, - - jsonFixturesPreprocessor: { - // Strip full file system part from the file path / fixture name - stripPrefix: ".+/", - // Change the global fixtures variable name - variableName: "__test_jsons" - }, - - // Start these browsers. - // Available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - browsers: [], - - // Test results reporter to use. - // Possible values: 'dots', 'progress'. - // Available reporters: https://npmjs.org/browse/keyword/karma-reporter - reporters: ["progress"], - - // Disallow Karma launching multiple browsers at the same time - concurrency: 1 - - }); -}; diff --git a/karma/configs/chrome.js b/karma/configs/chrome.js deleted file mode 100644 index c2c96732..00000000 --- a/karma/configs/chrome.js +++ /dev/null @@ -1,22 +0,0 @@ -"use strict"; - -let setCommonOptions = require("./base"); - -module.exports = function (config) { - - setCommonOptions(config); - - config.set({ - - browsers: ["ChromeNoSandbox"], - - // Chrome crashes the page when opened in a sandbox - customLaunchers: { - ChromeNoSandbox: { - base: "Chrome", - flags: ["--no-sandbox"] - } - }, - - }); -}; diff --git a/karma/configs/firefox.js b/karma/configs/firefox.js deleted file mode 100644 index a4c84327..00000000 --- a/karma/configs/firefox.js +++ /dev/null @@ -1,12 +0,0 @@ -"use strict"; - -let setCommonOptions = require("./base"); - -module.exports = function (config) { - - setCommonOptions(config); - - config.set({ - browsers: ["Firefox"] - }); -}; diff --git a/karma/configs/headless.js b/karma/configs/headless.js deleted file mode 100644 index c2591e3c..00000000 --- a/karma/configs/headless.js +++ /dev/null @@ -1,29 +0,0 @@ -"use strict"; - -let setCommonOptions = require("./base"); - -module.exports = function (config) { - - setCommonOptions(config); - - config.set({ - - browsers: ["ChromeHeadless"], - - customLaunchers: { - ChromeHeadless: { - base: "Chrome", - flags: [ - "--headless", - "--disable-gpu", - "--no-sandbox", - // Without a remote debugging port, Google Chrome exits immediately. - "--remote-debugging-port=9222" - ] - } - }, - - singleRun: true - - }); -}; diff --git a/karma/configs/ie.js b/karma/configs/ie.js deleted file mode 100644 index 53a1e5c5..00000000 --- a/karma/configs/ie.js +++ /dev/null @@ -1,12 +0,0 @@ -"use strict"; - -let setCommonOptions = require("./base"); - -module.exports = function (config) { - - setCommonOptions(config); - - config.set({ - browsers: ["IE"], - }); -}; diff --git a/karma/fixtures/clarity.fixture.html b/karma/fixtures/clarity.fixture.html deleted file mode 100644 index 099997ed..00000000 --- a/karma/fixtures/clarity.fixture.html +++ /dev/null @@ -1,17 +0,0 @@ - -
-

Clarity

-
- -

- The Clarity project contains code for analyzing users' interactions with webpages. -

-

- For more information, check out: clarity project. -

-
-
diff --git a/karma/setup/clarity.ts b/karma/setup/clarity.ts deleted file mode 100644 index 65d1f37d..00000000 --- a/karma/setup/clarity.ts +++ /dev/null @@ -1,148 +0,0 @@ -// @ts-ignore: 'merge' implicityly has 'any' type. There are no typings for 'merge' -import * as merge from "merge"; - -import { UploadCallback } from "@clarity-types/core"; -import { ClarityJs, IConfig } from "@clarity-types/index"; -import { setRealTimeout } from "./proxyapis/jasmineclock"; -import { publishAsync, publishSync, PubSubEvents, waitFor } from "./pubsub"; -import { getFullImpressionWatchResult, resetWatcher } from "./watch"; - -export interface IStartClarityOptions { - flushInitialActivity: boolean; -} - -declare var clarity: typeof ClarityJs; - -const uploadDelay: number = 500; -const defaultStartOptions: IStartClarityOptions = { - flushInitialActivity: true -}; - -let activeConfig: Partial = null; - -export async function startClarity(_config?: Partial, _startOptions?: IStartClarityOptions): Promise { - const testConfig: IConfig = merge(true, _config); - testConfig.delay = uploadDelay; - - const _uploadHandler = testConfig.uploadHandler; - testConfig.uploadHandler = (payload: string, onSuccess?: UploadCallback, onFailure?: UploadCallback): void => { - testUploadHandler(payload, onSuccess, onFailure); - if (_uploadHandler) { - _uploadHandler(payload, onSuccess, onFailure); - } - }; - - resetWatcher(); - activeConfig = testConfig; - clarity.start(testConfig); - - const startOptions = merge(true, defaultStartOptions, _startOptions); - if (startOptions.flushInitialActivity) { - await flushInitialActivity(); - } -} - -export function stopClarity(): void { - clarity.stop(); -} - -export async function restartClarity( - _config?: Partial, - _startOptions?: IStartClarityOptions, - beforeStart?: () => void -): Promise { - stopClarity(); - if (beforeStart) { - beforeStart(); - } - await startClarity(_config, _startOptions); -} - -export function triggerClarity(key?: string): void { - clarity.trigger(key || "testtrigger"); -} - -export async function triggerClarityAndWaitForUpload(): Promise { - return new Promise((resolve: any, reject: any): void => { - waitFor(PubSubEvents.UPLOAD).then(() => resolve()); - triggerClarity(); - }); -} - -export function triggerMutationEvent(attrs?: { [key: string]: string }): HTMLElement { - const triggerElement = document.createElement("eventtrigger"); - if (attrs) { - const attrNames = Object.keys(attrs); - for (const name of attrNames) { - const val = attrs[name]; - triggerElement.setAttribute(name, val); - } - } - document.body.appendChild(triggerElement); - return triggerElement; -} - -export async function triggerMutationEventAndWaitForUpload( - attrs?: { [key: string]: string } -): Promise { - return new Promise((resolve: any, reject: any): void => { - const triggerElem = triggerMutationEvent(attrs); - waitFor(PubSubEvents.MUTATION).then(() => { - waitFor(PubSubEvents.UPLOAD).then(() => resolve(triggerElem)); - jasmine.clock().tick(activeConfig.delay); - }); - }); -} - -export function getActiveConfig(): Partial { - return activeConfig; -} - -export function getVersion(): string { - return clarity.version; -} - -function testUploadHandler(payload: string, onSuccess?: UploadCallback, onFailure?: UploadCallback): void { - publishSync(PubSubEvents.SYNC_AFTER_UPLOAD, payload); - publishAsync(PubSubEvents.UPLOAD, payload); -} - -async function flushInitialActivity(): Promise { - return new Promise((resolve: any, reject: any): void => { - function tryResolve(): void { - hasActivity() - .then((active: boolean) => { - if (active) { - jasmine.clock().tick(100000); - const nextActivityIndicator = ( - activeConfig.backgroundMode - ? PubSubEvents.WORKER_COMPRESSED_BATCH_MESSAGE - : PubSubEvents.UPLOAD - ); - waitFor(nextActivityIndicator).then(tryResolve); - } else { - resolve(); - } - }); - } - tryResolve(); - }); -} - -async function hasActivity(): Promise { - return new Promise((resolve: any, reject: any): void => { - const allWatch = getFullImpressionWatchResult(); - const processedEvents = activeConfig.backgroundMode ? allWatch.compressedEvents : allWatch.sentEvents; - if (allWatch.coreEvents.length === processedEvents.length) { - // Trigger any timeout'ed activity in Clarity and then set a real timeout to give our - // async PubSub notifications a chance to report any activity if it happened - jasmine.clock().tick(100000); - setRealTimeout(() => { - const refreshAllWatch = getFullImpressionWatchResult(); - resolve(!(refreshAllWatch.coreEvents.length === allWatch.coreEvents.length)); - }, 0); - } else { - resolve(true); - } - }); -} diff --git a/karma/setup/mocks/error.ts b/karma/setup/mocks/error.ts deleted file mode 100644 index 07c63920..00000000 --- a/karma/setup/mocks/error.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const MockErrorMessage: string = "MockErrorMessage"; - -export const MockErrorInit: ErrorEventInit = { - message: "MockErrorEventMessage", - filename: "mockfile.ts", - lineno: 1111111, - colno: 2222222 -}; diff --git a/karma/setup/mocks/event.ts b/karma/setup/mocks/event.ts deleted file mode 100644 index f6c98dc3..00000000 --- a/karma/setup/mocks/event.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { IEnvelope, IEvent } from "@clarity-types/core"; - -export const MockEventName = "ClarityTestMockEvent"; - -export function createMockEvent(eventName?: string): IEvent { - let mockEvent: IEvent = { - id: -1, - state: {}, - time: -1, - type: eventName || MockEventName - }; - return mockEvent; -} - -export function createMockEnvelope(sequenceNumber?: number): IEnvelope { - return { - clarityId: "MockClarityId", - impressionId: "MockImpressionId", - projectId: "MockProjectId", - sequenceNumber: sequenceNumber >= 0 ? sequenceNumber : -1, - time: -1, - url: window.location.toString(), - version: "0.0.0" - }; -} diff --git a/karma/setup/mocks/performance.ts b/karma/setup/mocks/performance.ts deleted file mode 100644 index 77d5f5db..00000000 --- a/karma/setup/mocks/performance.ts +++ /dev/null @@ -1,130 +0,0 @@ -type Mutable = { - -readonly[P in keyof T]: T[P] -}; - -type IMockPerformanceTiming = Partial>; -type IMockPerformanceEntry = Partial>; -type IMockPerformanceResourceTiming = Partial>; - -export interface IMockPerformance { - timing: IMockPerformanceTiming; - addEntry(entry: IMockPerformanceEntry): void; - getEntriesByType(type: string): IMockPerformanceEntry[]; - clearResourceTimings(): void; -} - -export function createMockPerformanceObject(): IMockPerformance { - let entries: IMockPerformanceEntry[] = []; - return { - timing: createMockPerformanceTiming(), - addEntry(entry: IMockPerformanceEntry): void { - entries.push(entry); - }, - getEntriesByType(type: string): IMockPerformanceEntry[] { - return entries.filter((entry: IMockPerformanceEntry): boolean => entry.entryType === type); - }, - clearResourceTimings(): void { - entries = entries.filter((entry: IMockPerformanceEntry): boolean => entry.entryType !== "resource"); - } - }; -} - -export function createMockPerformanceTiming(): IMockPerformanceTiming { - return { - connectEnd: 1548794791246, - connectStart: 1548794791216, - domComplete: 1548794793320, - domContentLoadedEventEnd: 1548794792866, - domContentLoadedEventStart: 1548794792596, - domInteractive: 1548794792596, - domLoading: 1548794791381, - domainLookupEnd: 1548794791216, - domainLookupStart: 1548794791207, - fetchStart: 1548794791203, - loadEventEnd: 1548794793323, - loadEventStart: 1548794793320, - navigationStart: 1548794791202, - redirectEnd: 0, - redirectStart: 0, - requestStart: 1548794791246, - responseEnd: 1548794792046, - responseStart: 1548794791354, - secureConnectionStart: 1548794791225, - unloadEventEnd: 1548794791364, - unloadEventStart: 1548794791363 - }; -} - -export function createMockPerformanceResourceTimings(): IMockPerformanceResourceTiming[] { - return [ - { - connectEnd: 276.8000001087785, - connectStart: 276.8000001087785, - decodedBodySize: 52853, - domainLookupEnd: 276.8000001087785, - domainLookupStart: 276.8000001087785, - duration: 335.1000000257045, - encodedBodySize: 12851, - entryType: "resource", - fetchStart: 276.8000001087785, - initiatorType: "script", - name: "https://mockperformance.test/resource/1", - nextHopProtocol: "h2", - redirectEnd: 0, - redirectStart: 0, - requestStart: 506.2000001780689, - responseEnd: 611.900000134483, - responseStart: 510.5000000912696, - secureConnectionStart: 0, - startTime: 276.8000001087785, - transferSize: 12948, - workerStart: 0 - }, - { - connectEnd: 287.40000003017485, - connectStart: 287.40000003017485, - decodedBodySize: 701397, - domainLookupEnd: 287.40000003017485, - domainLookupStart: 287.40000003017485, - duration: 553.7000000476837, - encodedBodySize: 158455, - entryType: "resource", - fetchStart: 287.40000003017485, - initiatorType: "script", - name: "https://mockperformance.test/resource/2", - nextHopProtocol: "h2", - redirectEnd: 0, - redirectStart: 0, - requestStart: 609.6000000834465, - responseEnd: 841.1000000778586, - responseStart: 647.1000001765788, - secureConnectionStart: 0, - startTime: 287.40000003017485, - transferSize: 158658, - workerStart: 0 - }, - { - connectEnd: 2888.3000002242625, - connectStart: 2888.3000002242625, - decodedBodySize: 499, - domainLookupEnd: 2888.3000002242625, - domainLookupStart: 2888.3000002242625, - duration: 395.5999999307096, - encodedBodySize: 499, - entryType: "resource", - fetchStart: 2888.3000002242625, - initiatorType: "img", - name: "https://mockperformance.test/resource/3", - nextHopProtocol: "h2", - redirectEnd: 0, - redirectStart: 0, - requestStart: 2898.100000107661, - responseEnd: 3283.900000154972, - responseStart: 3282.0000001229346, - secureConnectionStart: 0, - startTime: 2888.3000002242625, - transferSize: 812, - workerStart: 0 - } - ]; -} diff --git a/karma/setup/page.ts b/karma/setup/page.ts deleted file mode 100644 index 94f49cc9..00000000 --- a/karma/setup/page.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { IConfig } from "@clarity-types/config"; -import { IStartClarityOptions, startClarity, stopClarity } from "./clarity"; -import { installEventListenerProxies, uninstallEventListenerProxies } from "./proxyapis/eventlistener"; -import { installJasmineClock, uninstallJasmineClock } from "./proxyapis/jasmineclock"; -import { installMutationObserverProxy, uninstallMutationObserverProxy } from "./proxyapis/mutationobserver"; -import { installPerformanceProxy, uninstallPerformanceProxy } from "./proxyapis/performance"; -import { installWorkerProxy, uninstallWorkerProxy } from "./proxyapis/worker"; -import { revokeAllMessages, unsubscribeAll } from "./pubsub"; -import { stopWatching } from "./watch"; - -export async function setupPage(done: DoneFn, _config?: IConfig, _startOptions?: IStartClarityOptions): Promise { - // Relative to karma config location - fixture.setBase("karma/fixtures"); - fixture.load("clarity.fixture.html"); - - // Install API proxies - installJasmineClock(); - installMutationObserverProxy(); - installEventListenerProxies(); - installWorkerProxy(); - installPerformanceProxy(); - - await startClarity(_config, _startOptions); - done(); -} - -export function cleanupPage(): void { - stopWatching(); - stopClarity(); - - // Uninstall API proxies - uninstallJasmineClock(); - uninstallMutationObserverProxy(); - uninstallEventListenerProxies(); - uninstallWorkerProxy(); - uninstallPerformanceProxy(); - - // Reset PubSub - unsubscribeAll(); - revokeAllMessages(); - - fixture.cleanup(); -} diff --git a/karma/setup/proxyapis/eventlistener.ts b/karma/setup/proxyapis/eventlistener.ts deleted file mode 100644 index 5237e1e5..00000000 --- a/karma/setup/proxyapis/eventlistener.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { publishAsync, PubSubEvents } from "@karma/setup/pubsub"; - -let _addEventListener: typeof Node.prototype.addEventListener = null; -let _removeEventListener: typeof Node.prototype.removeEventListener = null; - -const CLARITY_TEST_PROXY_LISTENER_ATTRIBUTE = "CLARITY_TEST_PROXY_LISTENER"; - -export function installEventListenerProxies(): void { - installAddEventListenerProxy(); - installRemoveEventListenerProxy(); -} - -export function uninstallEventListenerProxies(): void { - uninstallAddEventListenerProxy(); - uninstallRemoveEventListenerProxy(); -} - -function installAddEventListenerProxy(): void { - if (_addEventListener === null) { - _addEventListener = Node.prototype.addEventListener; - } - function proxyAddEventListener( - type: string, - listener: EventListenerOrEventListenerObject | null, - options?: boolean | AddEventListenerOptions - ): void { - let proxyListener = (evt: Event): void => { - if (listener) { - (listener as EventListenerObject).handleEvent - ? (listener as EventListenerObject).handleEvent(evt) - : (listener as EventListener)(evt); - } - publishAsync(getTopicFromEventType(evt.type), null); - }; - proxyListener = listener[CLARITY_TEST_PROXY_LISTENER_ATTRIBUTE] || proxyListener; - listener[CLARITY_TEST_PROXY_LISTENER_ATTRIBUTE] = proxyListener; - - _addEventListener.call(this, type, proxyListener, options); - } - Node.prototype.addEventListener = proxyAddEventListener; -} - -function installRemoveEventListenerProxy(): void { - if (_removeEventListener === null) { - _removeEventListener = Node.prototype.removeEventListener; - } - function proxyRemoveEventListener( - type: string, - listener: EventListenerOrEventListenerObject | null, - options?: boolean | AddEventListenerOptions - ): void { - const proxyListener = listener[CLARITY_TEST_PROXY_LISTENER_ATTRIBUTE] || listener; - _removeEventListener.call(this, type, proxyListener, options); - } - Node.prototype.removeEventListener = proxyRemoveEventListener; -} - -function uninstallAddEventListenerProxy(): void { - if (_addEventListener !== null) { - Node.prototype.addEventListener = _addEventListener; - _addEventListener = null; - } -} - -function uninstallRemoveEventListenerProxy(): void { - if (_removeEventListener !== null) { - Node.prototype.removeEventListener = _removeEventListener; - _removeEventListener = null; - } -} - -function getTopicFromEventType(type: string): PubSubEvents { - switch (type) { - case "click": - return PubSubEvents.CLICK; - case "scroll": - return PubSubEvents.SCROLL; - case "input": - return PubSubEvents.INPUT; - case "change": - return PubSubEvents.CHANGE; - default: - return PubSubEvents.UNKNOWN; - } -} diff --git a/karma/setup/proxyapis/jasmineclock.ts b/karma/setup/proxyapis/jasmineclock.ts deleted file mode 100644 index 6eb484bd..00000000 --- a/karma/setup/proxyapis/jasmineclock.ts +++ /dev/null @@ -1,24 +0,0 @@ -const w: any = window; -const _setTimeout = window.setTimeout; -const _clearTimeout = window.clearTimeout; - -export function installJasmineClock(): void { - jasmine.clock().install(); -} - -export function uninstallJasmineClock(): void { - jasmine.clock().uninstall(); -} - -export const setRealTimeout: typeof window.setTimeout = (...args: any[]): number => { - w._setTimeout = _setTimeout; - const retval = w._setTimeout(...args); - delete w._setTimeout; - return retval; -}; - -export const clearRealTimeout: typeof window.clearTimeout = (...args: any[]): void => { - w._clearTimeout = _clearTimeout; - w._clearTimeout(...args); - delete w._clearTimeout; -}; diff --git a/karma/setup/proxyapis/mutationobserver.ts b/karma/setup/proxyapis/mutationobserver.ts deleted file mode 100644 index b8afd15c..00000000 --- a/karma/setup/proxyapis/mutationobserver.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { publishAsync, PubSubEvents } from "@karma/setup/pubsub"; - -let _MutationObserver: typeof MutationObserver = null; - -export const installMutationObserverProxy = (): void => { - if (_MutationObserver === null) { - _MutationObserver = MutationObserver; - } - MutationObserver = ((callback: MutationCallback): typeof MutationObserver => { - return new _MutationObserver( - (mutations: MutationRecord[], observer: MutationObserver): void => { - callback(mutations, observer); - publishAsync(PubSubEvents.MUTATION, null); - } - ) as any as typeof MutationObserver; - }) as any as typeof MutationObserver; -}; - -export const uninstallMutationObserverProxy = (): void => { - if (_MutationObserver !== null) { - MutationObserver = _MutationObserver; - _MutationObserver = null; - } -}; diff --git a/karma/setup/proxyapis/performance.ts b/karma/setup/proxyapis/performance.ts deleted file mode 100644 index a4cb2787..00000000 --- a/karma/setup/proxyapis/performance.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { createMockPerformanceObject } from "@karma/setup/mocks/performance"; - -const w: any = window; -let _performance: Performance = null; - -export function installPerformanceProxy(): void { - if (_performance === null) { - _performance = performance; - w.performance = createMockPerformanceObject(); - } -} - -export function uninstallPerformanceProxy(): void { - if (_performance !== null) { - w.performance = _performance; - _performance = null; - } -} diff --git a/karma/setup/proxyapis/worker.ts b/karma/setup/proxyapis/worker.ts deleted file mode 100644 index 3b06cab7..00000000 --- a/karma/setup/proxyapis/worker.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { IAddEventMessage, ICompressedBatchMessage, IWorkerMessage, WorkerMessageType } from "@clarity-types/compressionworker"; -import { publishAsync, publishSync, PubSubEvents } from "@karma/setup/pubsub"; -import { setRealTimeout } from "./jasmineclock"; - -interface IWorkerProxy extends Worker { - _postMessage: typeof Worker.prototype.postMessage; - _onmessage: typeof Worker.prototype.onmessage; -} - -let _Worker: typeof Worker = null; - -export const installWorkerProxy = (): void => { - if (_Worker === null) { - _Worker = Worker; - } - Worker = (function(aURL: string, options: WorkerOptions, ig: string): Worker { - const instance: IWorkerProxy = new _Worker(aURL, options) as IWorkerProxy; - instance._postMessage = instance.postMessage; - instance.postMessage = (msg: IWorkerMessage): void => { - instance._postMessage(msg); - processWorkerMessage(msg); - }; - - // In Clarity code Worker's 'ongmessage' handler is assigned synchronously immediately after its instantiation - // This means that at this time we don't have access to it and we need to come back and proxy it once its set. - // NOTE: This won't handle more complex scenarios where 'onmessage' might be set or changed asynchronously at - // some point in the future, however it's not something that's relevant to our use case right now. When/if such - // complexity comes along, we can use object mutation detection libraries to wrap updated handlers as well. - setRealTimeout(() => { - instance._onmessage = instance.onmessage; - instance.onmessage = (msgEvt: MessageEvent): void => { - instance._onmessage(msgEvt); - processWorkerMessage(msgEvt.data); - }; - }, 0); - - return instance; - }) as any as typeof Worker; -}; - -export const uninstallWorkerProxy = (): void => { - if (_Worker !== null) { - Worker = _Worker; - _Worker = null; - } -}; - -function processWorkerMessage(message: IWorkerMessage): void { - switch (message.type) { - - // Page ===> Worker - case WorkerMessageType.AddEvent: - let addEventMsg = message as IAddEventMessage; - publishSync(PubSubEvents.SYNC_AFTER_WORKER_ADD_EVENT_MESSAGE, addEventMsg); - publishAsync(PubSubEvents.WORKER_ADD_EVENT_MESSAGE, addEventMsg); - break; - case WorkerMessageType.ForceCompression: - publishAsync(PubSubEvents.WORKER_FORCE_COMPRESSION_MESSAGE, message); - break; - - // Page <=== Worker - case WorkerMessageType.CompressedBatch: - let uploadMsg = message as ICompressedBatchMessage; - publishSync(PubSubEvents.SYNC_AFTER_WORKER_COMPRESSED_BATCH_MESSAGE, message); - publishAsync(PubSubEvents.WORKER_COMPRESSED_BATCH_MESSAGE, uploadMsg); - break; - default: - break; - } -} diff --git a/karma/setup/pubsub.ts b/karma/setup/pubsub.ts deleted file mode 100644 index 8e9a7c14..00000000 --- a/karma/setup/pubsub.ts +++ /dev/null @@ -1,103 +0,0 @@ -import * as PubSub from "pubsub-js"; - -import { clearRealTimeout, setRealTimeout } from "./proxyapis/jasmineclock"; - -const _publishSync = PubSub.publishSync; - -// Development option per package documentation: -// https://www.npmjs.com/package/pubsub-js -(PubSub as any).immediateExceptions = true; - -// Our tests are instrumented with Jasmine clock, which means that async publish wouldn't trigger until we manually -// move the clock forward. However, jasmine clock tick is synchronous, so then publish would work as publishSync. -// One should use an explicit publishAsync utility function provided below to achieve true async behavior. -(PubSub as any).publish = (): boolean => { - throw new Error("Direct use of PubSub 'publish' is disallowed."); -}; - -// Synchronous publishing can get out of control very easily and result in some complicated bugs - avoid using it -// If it is absolutely required, use an explicity publishSync utility function provided below. -(PubSub as any).publishSync = (): boolean => { - throw new Error("Direct use of PubSub 'publishSync' is disallowed."); -}; - -export enum PubSubEvents { - - // Page ====> Worker messages - WORKER_ADD_EVENT_MESSAGE, - WORKER_FORCE_COMPRESSION_MESSAGE, - SYNC_AFTER_WORKER_ADD_EVENT_MESSAGE, - - // Page <==== Worker messages - WORKER_COMPRESSED_BATCH_MESSAGE, - SYNC_AFTER_WORKER_COMPRESSED_BATCH_MESSAGE, - - // End output monitoring - UPLOAD, - SYNC_AFTER_UPLOAD, - - // Browser callbacks - MUTATION, - CLICK, - SCROLL, - INPUT, - CHANGE, - UNKNOWN -} - -const timeouts: { [key: string]: number } = {}; - -// WARNING: Be careful with this and use only when absolutely required! -// PubSub docs say "USE WITH CAUTION, HERE BE DRAGONS!!!". I saw them - they are marvelous, but terrifying creatures. -// If you still end up using it, try to keep your sync subscribers as dumb as possible to perform some -// simple task and return immediately without calling other functions (ideally). -export function publishSync(message: any, data: any): void { - _publishSync.apply(PubSub, [message, data]); -} - -export function publishAsync(message: any, data: any): void { - const timeoutId = setRealTimeout(() => { - delete timeouts[timeoutId]; - _publishSync.apply(PubSub, [message, data]); - }, 0); - timeouts[timeoutId] = timeoutId; -} - -export function unsubscribeAll(): void { - PubSub.clearAllSubscriptions(); -} - -export function revokeAllMessages(): void { - const timeoutIds = Object.keys(timeouts); - for (const timeoutIdStr of timeoutIds) { - const timeoutId = timeouts[timeoutIdStr]; - clearRealTimeout(timeoutId); - delete timeouts[timeoutIdStr]; - } -} - -export async function waitFor(message: any, abandonAfterMs?: number): Promise { - return new Promise((resolve: any, reject: any): void => { - function onMessage(): void { - PubSub.unsubscribe(subToken); - clearRealTimeout(abandonTimeout); - resolve(true); - } - function onTimeout(): void { - PubSub.unsubscribe(subToken); - resolve(false); - } - let abandonTimeout: number = null; - const subToken = PubSub.subscribe(message, onMessage); - if (abandonAfterMs !== undefined) { - abandonTimeout = setRealTimeout(onTimeout, abandonAfterMs); - } - }); -} - -// Yielding thread allows subscribers receive and process async messages -export async function yieldThread(): Promise { - return new Promise((resolve: any, reject: any): void => { - setRealTimeout(resolve, 0); - }); -} diff --git a/karma/setup/testasync.ts b/karma/setup/testasync.ts deleted file mode 100644 index cfe10e10..00000000 --- a/karma/setup/testasync.ts +++ /dev/null @@ -1,9 +0,0 @@ -// There is a known issue where an (assertion) error within an async test doesn't cause the test to fail -// Instead, it causes a test timeout. Wrapping async test function with the code below lets us fail test -// immediately on (assertion) error, although stack trace and error message are altered slightly -// https://github.com/mochajs/mocha/issues/1128 -export function testAsync(testFn: (done: DoneFn) => Promise): (done: DoneFn) => void { - return (done: DoneFn): void => { - testFn(done).catch(done); - }; -} diff --git a/karma/setup/uncompress.ts b/karma/setup/uncompress.ts deleted file mode 100644 index 953e16bb..00000000 --- a/karma/setup/uncompress.ts +++ /dev/null @@ -1,190 +0,0 @@ -/* Credit: http://pieroxy.net/blog/pages/lz-string/index.html */ - -// Excluding 3rd party code from tslint -/* tslint:disable */ -export default function(compressed: string) { - // use decompressFromBase64 to parallel the compress function - var keyStrBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; - if (compressed == null) return ""; - if (compressed == "") return null; - var f = String.fromCharCode; - var length = compressed.length; - var resetValue = 32; - // @ts-ignore: Implicit 'any' type - var getNextValue = function(index) { return getBaseValue(keyStrBase64, compressed.charAt(index)); } - var baseReverseDic = {}; - - // @ts-ignore: Implicit 'any' type - function getBaseValue(alphabet, character) { - if (!baseReverseDic[alphabet]) { - baseReverseDic[alphabet] = {}; - for (var i = 0; i < alphabet.length; i++) { - baseReverseDic[alphabet][alphabet.charAt(i)] = i; - } - } - return baseReverseDic[alphabet][character]; - } - - var dictionary = [], - // Not modifying 3rd party code - suppressing ts error instead. - // @ts-ignore: Unused local variable - next, - enlargeIn = 4, - dictSize = 4, - numBits = 3, - entry = "", - result = [], - i, - w, - bits, resb, maxpower, power, - c, - data = { val: getNextValue(0), position: resetValue, index: 1 }; - - for (i = 0; i < 3; i += 1) { - dictionary[i] = i; - } - - bits = 0; - maxpower = Math.pow(2, 2); - power = 1; - while (power != maxpower) { - resb = data.val & data.position; - data.position >>= 1; - if (data.position == 0) { - data.position = resetValue; - data.val = getNextValue(data.index++); - } - bits |= (resb > 0 ? 1 : 0) * power; - power <<= 1; - } - - switch (next = bits) { - case 0: - bits = 0; - maxpower = Math.pow(2, 8); - power = 1; - while (power != maxpower) { - resb = data.val & data.position; - data.position >>= 1; - if (data.position == 0) { - data.position = resetValue; - data.val = getNextValue(data.index++); - } - bits |= (resb > 0 ? 1 : 0) * power; - power <<= 1; - } - c = f(bits); - break; - case 1: - bits = 0; - maxpower = Math.pow(2, 16); - power = 1; - while (power != maxpower) { - resb = data.val & data.position; - data.position >>= 1; - if (data.position == 0) { - data.position = resetValue; - data.val = getNextValue(data.index++); - } - bits |= (resb > 0 ? 1 : 0) * power; - power <<= 1; - } - c = f(bits); - break; - case 2: - return ""; - } - dictionary[3] = c; - w = c; - result.push(c); - while (true) { - if (data.index > length) { - return ""; - } - - bits = 0; - maxpower = Math.pow(2, numBits); - power = 1; - while (power != maxpower) { - resb = data.val & data.position; - data.position >>= 1; - if (data.position == 0) { - data.position = resetValue; - data.val = getNextValue(data.index++); - } - bits |= (resb > 0 ? 1 : 0) * power; - power <<= 1; - } - - switch (c = bits) { - case 0: - bits = 0; - maxpower = Math.pow(2, 8); - power = 1; - while (power != maxpower) { - resb = data.val & data.position; - data.position >>= 1; - if (data.position == 0) { - data.position = resetValue; - data.val = getNextValue(data.index++); - } - bits |= (resb > 0 ? 1 : 0) * power; - power <<= 1; - } - - dictionary[dictSize++] = f(bits); - c = dictSize - 1; - enlargeIn--; - break; - case 1: - bits = 0; - maxpower = Math.pow(2, 16); - power = 1; - while (power != maxpower) { - resb = data.val & data.position; - data.position >>= 1; - if (data.position == 0) { - data.position = resetValue; - data.val = getNextValue(data.index++); - } - bits |= (resb > 0 ? 1 : 0) * power; - power <<= 1; - } - dictionary[dictSize++] = f(bits); - c = dictSize - 1; - enlargeIn--; - break; - case 2: - return result.join(''); - } - - if (enlargeIn == 0) { - enlargeIn = Math.pow(2, numBits); - numBits++; - } - - if (dictionary[c]) { - // @ts-ignore: Type 'string | number' is not assignable to type 'string' - entry = dictionary[c]; - } else { - if (c === dictSize) { - entry = w + w.charAt(0); - } else { - return null; - } - } - result.push(entry); - - // Add w+entry[0] to the dictionary. - dictionary[dictSize++] = w + entry.charAt(0); - enlargeIn--; - - w = entry; - - if (enlargeIn == 0) { - enlargeIn = Math.pow(2, numBits); - numBits++; - } - - } -} diff --git a/karma/setup/watch.ts b/karma/setup/watch.ts deleted file mode 100644 index e69eda22..00000000 --- a/karma/setup/watch.ts +++ /dev/null @@ -1,91 +0,0 @@ -import uncompress from "./uncompress"; - -import { IAddEventMessage, ICompressedBatchMessage } from "@clarity-types/compressionworker"; -import { IEvent, IEventArray, IPayload } from "@clarity-types/core"; -import { decode } from "@src/converters/convert"; -import { SchemaManager } from "@src/converters/schema"; -import { PubSubEvents } from "./pubsub"; - -interface IWatchResult { - coreEvents: IEvent[]; - compressedEvents: IEvent[]; - sentEvents: IEvent[]; -} - -let schemas: SchemaManager = new SchemaManager(); -let watchResult: IWatchResult = null; -let fullImpression: IWatchResult = null; -let addEventMessageSub: any = null; -let compressedBatchSub: any = null; -let uploadSub: any = null; -let watching: boolean = false; - -export function watch(): any { - if (watching) { - throw new Error("Already watching"); - } else { - watchResult = { - coreEvents: [], - compressedEvents: [], - sentEvents: [] - }; - watching = true; - } -} - -export function stopWatching(): IWatchResult { - watching = false; - return watchResult; -} - -export function getFullImpressionWatchResult(): IWatchResult { - return JSON.parse(JSON.stringify(fullImpression)); -} - -export function resetWatcher(): void { - fullImpression = { - coreEvents: [], - compressedEvents: [], - sentEvents: [] - }; - schemas.reset(); - - // Resubscribe to all relevant topics - PubSub.unsubscribe(addEventMessageSub); - PubSub.unsubscribe(compressedBatchSub); - PubSub.unsubscribe(uploadSub); - - addEventMessageSub = PubSub.subscribe(PubSubEvents.SYNC_AFTER_WORKER_ADD_EVENT_MESSAGE, onAddEventMessage); - compressedBatchSub = PubSub.subscribe(PubSubEvents.SYNC_AFTER_WORKER_COMPRESSED_BATCH_MESSAGE, onCompressedBatch); - uploadSub = PubSub.subscribe(PubSubEvents.SYNC_AFTER_UPLOAD, onUpload); -} - -export function onAddEventMessage(pubSubMessage: any, workerMessage: IAddEventMessage): void { - const decodedEvent: IEvent = decode(workerMessage.event, schemas); - fullImpression.coreEvents.push(decodedEvent); - if (watching) { - watchResult.coreEvents.push(decodedEvent); - } -} - -export function onCompressedBatch(pubSubMessage: any, workerMessage: ICompressedBatchMessage): void { - const compressedEvents = workerMessage.rawData.events.map((encodedEvent: IEventArray): IEvent => decode(encodedEvent, schemas)); - fullImpression.compressedEvents.push(...compressedEvents); - if (watching) { - watchResult.compressedEvents.push(...compressedEvents); - } -} - -export function onUpload(message: any, data: any): void { - const compressedPayload = data; - const payload: IPayload = JSON.parse(uncompress(compressedPayload)); - const decodedEvents = payload.events.map((encodedEvent: IEventArray): IEvent => decode(encodedEvent, schemas)); - fullImpression.sentEvents.push(...decodedEvents); - if (watching) { - watchResult.sentEvents.push(...decodedEvents); - } -} - -export function filterEventsByType(events: IEvent[], type: string): IEvent[] { - return events.filter((event: IEvent): boolean => event.type === type); -} diff --git a/karma/tests/_setup/asyncstart.ts b/karma/tests/_setup/asyncstart.ts deleted file mode 100644 index 0c52c8a8..00000000 --- a/karma/tests/_setup/asyncstart.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { restartClarity } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { setRealTimeout } from "@karma/setup/proxyapis/jasmineclock"; -import { testAsync } from "@karma/setup/testasync"; -import { getFullImpressionWatchResult, stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Setup: Asynchronous clarity start tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that intial activity events have been generated and sent on async start", (done: DoneFn) => { - const fullImpressionWatch = getFullImpressionWatchResult(); - assert.isAbove(fullImpressionWatch.coreEvents.length, 0); - assert.isAbove(fullImpressionWatch.sentEvents.length, 0); - assert.equal(fullImpressionWatch.coreEvents.length, fullImpressionWatch.sentEvents.length); - done(); - }); - - it("checks that 'setupPage' flushes initial Clarity synchronous activity", (done: DoneFn) => { - watch(); - jasmine.clock().tick(10000000); - const events = stopWatching().coreEvents; - assert.equal(events.length, 0); - done(); - }); - - it("checks that 'setupPage' flushes initial Clarity asynchronous activity", (done: DoneFn) => { - watch(); - setRealTimeout(() => { - const events = stopWatching().coreEvents; - assert.equal(events.length, 0); - done(); - }, 1); - }); - - it("checks that 'setupPage' flushes all initial Clarity activity but nothing's sent in background mode", - testAsync(async (done: DoneFn) => { - await restartClarity({ backgroundMode: true }); - setRealTimeout(() => { - const allEvents = getFullImpressionWatchResult(); - assert.equal(allEvents.coreEvents.length, allEvents.compressedEvents.length); - assert.equal(allEvents.sentEvents.length, 0); - done(); - }, 1); - }) - ); - -}); diff --git a/karma/tests/_setup/pubsub.ts b/karma/tests/_setup/pubsub.ts deleted file mode 100644 index a82d4d17..00000000 --- a/karma/tests/_setup/pubsub.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Setup: PubSub Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that 'waitFor' async function works as intended", testAsync(async (done: DoneFn) => { - watch(); - document.body.appendChild(document.createElement("div")); - await waitFor(PubSubEvents.MUTATION); - const events = stopWatching().coreEvents; - assert.isAbove(events.length, 0); - done(); - })); - -}); diff --git a/karma/tests/_setup/syncstart.ts b/karma/tests/_setup/syncstart.ts deleted file mode 100644 index 548e734d..00000000 --- a/karma/tests/_setup/syncstart.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { getFullImpressionWatchResult } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Setup: Synchronous clarity start tests", () => { - - beforeEach((done: DoneFn) => setupPage(done, {}, { flushInitialActivity: false })); - afterEach(cleanupPage); - - it("checks that events have been generated but not sent yet on synchonous start", (done: DoneFn) => { - const fullImpressionWatch = getFullImpressionWatchResult(); - assert.isAbove(fullImpressionWatch.coreEvents.length, 0); - assert.equal(fullImpressionWatch.sentEvents.length, 0); - done(); - }); - -}); diff --git a/karma/tests/compressionworker.ts b/karma/tests/compressionworker.ts deleted file mode 100644 index be71a900..00000000 --- a/karma/tests/compressionworker.ts +++ /dev/null @@ -1,158 +0,0 @@ -import schemas from "@src/converters/schema"; - -import { IAddEventMessage, ITimestampedWorkerMessage, WorkerMessageType } from "@clarity-types/compressionworker"; -import { IEvent } from "@clarity-types/core"; -import { Instrumentation, IXhrErrorEventState } from "@clarity-types/instrumentation"; -import { createMockEnvelope, createMockEvent, MockEventName } from "@karma/setup/mocks/event"; -import { installWorkerProxy, uninstallWorkerProxy } from "@karma/setup/proxyapis/worker"; -import { PubSubEvents, unsubscribeAll, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { resetWatcher, stopWatching, watch } from "@karma/setup/watch"; -import { createCompressionWorker } from "@src/compressionworker"; -import { encode } from "@src/converters/convert"; -import { assert } from "chai"; - -// TODO: Remove src imports and modify compression worker to accept batchLimit as an argument -import { config } from "@src/config"; - -const InstrumentationEventName = "Instrumentation"; -const WorkerMessageWaitTime = 1000; - -describe("Compression Worker Tests", () => { - let worker: Worker = null; - - beforeEach(() => { - resetWatcher(); - installWorkerProxy(); - schemas.reset(); - worker = createTestWorker(); - }); - afterEach(() => { - worker.terminate(); - unsubscribeAll(); - uninstallWorkerProxy(); - }); - - it("validates that events are batched for upload correctly when total length is below the limit", - testAsync(async (done: DoneFn) => { - watch(); - let firstMockEventName = "FirstMockEvent"; - let firstMockEvent = createMockEvent(firstMockEventName); - let addEventMessage = createAddEventMessage(firstMockEvent); - worker.postMessage(addEventMessage); - - let secondMockEventName = "SecondMockEvent"; - let secondMockEvent = createMockEvent(secondMockEventName); - addEventMessage.event = encode(secondMockEvent); - worker.postMessage(addEventMessage); - - let forceCompressionMsg = createForceCompressionMessage(); - worker.postMessage(forceCompressionMsg); - await waitFor(PubSubEvents.WORKER_COMPRESSED_BATCH_MESSAGE); - - const compressedEvents = stopWatching().compressedEvents; - assert.equal(compressedEvents.length, 2); - assert.equal(compressedEvents[0].type, firstMockEventName); - assert.equal(compressedEvents[1].type, secondMockEventName); - done(); - - }) - ); - - it("validates that events are batched for upload correctly when total length is above the limit", - testAsync(async (done: DoneFn) => { - watch(); - let firstMockEventName = "FirstMockEvent"; - let firstMockEvent = createMockEvent(firstMockEventName); - let firstMockData = Array(Math.round(config.batchLimit * (2 / 3))).join("1"); - firstMockEvent.state = { data: firstMockData }; - let addEventMessage = createAddEventMessage(firstMockEvent); - worker.postMessage(addEventMessage); - - let secondMockEventName = "SecondMockEvent"; - let secondMockEvent = createMockEvent(secondMockEventName); - let secondMockData = Array(Math.round(config.batchLimit / 2)).join("2"); - secondMockEvent.state = { data: secondMockData }; - addEventMessage.event = encode(secondMockEvent); - worker.postMessage(addEventMessage); - - let forceCompressionMsg = createForceCompressionMessage(); - worker.postMessage(forceCompressionMsg); - await waitFor(PubSubEvents.WORKER_COMPRESSED_BATCH_MESSAGE); - - const firstBatchEvents = stopWatching().compressedEvents; - watch(); - await waitFor(PubSubEvents.WORKER_COMPRESSED_BATCH_MESSAGE); - - const secondBatchEvents = stopWatching().compressedEvents; - assert.equal(firstBatchEvents.length, 1); - assert.equal(firstBatchEvents[0].type, firstMockEventName); - assert.equal(secondBatchEvents.length, 1); - assert.equal(secondBatchEvents[0].type, secondMockEventName); - done(); - }) - ); - - it("validates that single event is sent when its own length is above the limit", testAsync(async (done: DoneFn) => { - watch(); - let mockEvent = createMockEvent(); - let mockEventData = Array(Math.round(config.batchLimit + 1)).join("1"); - mockEvent.state = { data: mockEventData }; - let addEventMessage = createAddEventMessage(mockEvent); - worker.postMessage(addEventMessage); - await waitFor(PubSubEvents.WORKER_COMPRESSED_BATCH_MESSAGE); - const compressedEvents = stopWatching().compressedEvents; - assert.equal(compressedEvents.length, 1); - assert.equal(compressedEvents[0].type, MockEventName); - done(); - })); - - it("validates that payloads consisting of a single XHR error event are not uploaded", - testAsync(async (done: DoneFn) => { - let mockXhrErrorEvent = createMockEvent(InstrumentationEventName); - let mockXhrErrorEventState = { - type: Instrumentation.XhrError - } as IXhrErrorEventState; - mockXhrErrorEvent.state = mockXhrErrorEventState; - const addEventMessage = createAddEventMessage(mockXhrErrorEvent); - // let addEventMessage: IAddEventMessage = { - // type: WorkerMessageType.AddEvent, - // time: -1, - // event: MockEventToArray(mockXhrErrorEvent), - // isXhrErrorEvent: true - // }; - worker.postMessage(addEventMessage); - - let forceCompressionMsg = createForceCompressionMessage(); - worker.postMessage(forceCompressionMsg); - const receivedCompressedBatch = await waitFor(PubSubEvents.UPLOAD, WorkerMessageWaitTime); - - receivedCompressedBatch - ? done.fail("Received batch consisting of a single XHR event") - : done(); - }) - ); - - function createTestWorker(): Worker { - return createCompressionWorker(createMockEnvelope(), () => { - // ignore - }); - } - - function createAddEventMessage(event: IEvent): IAddEventMessage { - let addEventMessage: IAddEventMessage = { - type: WorkerMessageType.AddEvent, - time: -1, - event: encode(event) - }; - return addEventMessage; - } - - function createForceCompressionMessage(): ITimestampedWorkerMessage { - let forceCompressionMessage: ITimestampedWorkerMessage = { - type: WorkerMessageType.ForceCompression, - time: -1 - }; - return forceCompressionMessage; - } -}); diff --git a/karma/tests/convert.ts b/karma/tests/convert.ts deleted file mode 100644 index 06e33263..00000000 --- a/karma/tests/convert.ts +++ /dev/null @@ -1,28 +0,0 @@ -import FromArray from "@src/converters/fromarray"; -import ToArray from "@src/converters/toarray"; - -import { IEvent } from "@clarity-types/core"; -import { createMockEvent } from "@karma/setup/mocks/event"; -import { assert } from "chai"; - -describe("Data Conversion Tests", () => { - - it("validates that conversion works", (done: DoneFn) => { - let mapObj = { - country: "USA", - cities: ["Seattle", "Boston"], - properties: { - zoom: 1.5 - } - }; - - let evt: IEvent = createMockEvent(); - evt.state = mapObj; - - let array = ToArray(evt); - let original = FromArray(array); - assert.equal(JSON.stringify(evt).length, JSON.stringify(original).length); - done(); - }); - -}); diff --git a/karma/tests/core/compression.ts b/karma/tests/core/compression.ts deleted file mode 100644 index f34ea66f..00000000 --- a/karma/tests/core/compression.ts +++ /dev/null @@ -1,18 +0,0 @@ -import uncompress from "@karma/setup/uncompress"; -import compress from "@src/compress"; - -import { assert } from "chai"; - -describe("Data Compression Tests", () => { - - it("validates that compression works", () => { - let str = "This is a string with some repetitions in the string."; - str += str + str; - let compressed = compress(str); - let uncompressed = uncompress(compressed); - assert.equal(compressed.length > 0 && compressed.length < str.length, true); - assert.equal(compressed === str, false); - assert.equal(uncompressed === str, true); - }); - -}); diff --git a/karma/tests/core/core.ts b/karma/tests/core/core.ts deleted file mode 100644 index 4a00f6e6..00000000 --- a/karma/tests/core/core.ts +++ /dev/null @@ -1,58 +0,0 @@ -import * as PubSub from "pubsub-js"; - -import { getActiveConfig, getVersion, triggerMutationEvent } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor, yieldThread } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { assert } from "chai"; - -describe("Core Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("validates that core.ts version matches package.json", (done: DoneFn) => { - let testJsons = window["__test_jsons"]; - let packageJson = testJsons && testJsons["package"]; - assert.equal(packageJson.version, getVersion()); - done(); - }); - - it("validates that force compression message is sent after config.delay milliseconds without new events", - testAsync(async (done: DoneFn) => { - triggerMutationEvent(); - await waitFor(PubSubEvents.MUTATION); - PubSub.subscribe(PubSubEvents.WORKER_FORCE_COMPRESSION_MESSAGE, done); - jasmine.clock().tick(getActiveConfig().delay); - }) - ); - - it("validates that force compression message is NOT sent after less than config.delay milliseconds without new events", - testAsync(async (done: DoneFn) => { - triggerMutationEvent(); - await waitFor(PubSubEvents.MUTATION); - jasmine.clock().tick(getActiveConfig().delay - 1); - PubSub.subscribe(PubSubEvents.WORKER_FORCE_COMPRESSION_MESSAGE, () => { done.fail("Received force compression message"); }); - await yieldThread(); - done(); - }) - ); - - it("validates that force compression timeout is reset with each new event", testAsync(async (done: DoneFn) => { - PubSub.subscribe(PubSubEvents.WORKER_FORCE_COMPRESSION_MESSAGE, () => { done.fail("Received force compression message"); }); - - triggerMutationEvent(); - await waitFor(PubSubEvents.MUTATION); - jasmine.clock().tick(getActiveConfig().delay - 1); - - // If force compression timeout wasn't reset between two events, second config.delay - 1 tick - // would trigger force compression and fail the test - triggerMutationEvent(); - await waitFor(PubSubEvents.MUTATION); - jasmine.clock().tick(getActiveConfig().delay - 1); - - await yieldThread(); - done(); - })); - -}); diff --git a/karma/tests/core/teardown.ts b/karma/tests/core/teardown.ts deleted file mode 100644 index 10eb9d63..00000000 --- a/karma/tests/core/teardown.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { restartClarity, stopClarity, triggerMutationEventAndWaitForUpload } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Core: Teardown Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("validates that pending events are sent on teardown", (done: DoneFn) => { - // Start clarity synchronously, so that initial events are "stuck" in compression workers when we call stopClarity() - restartClarity({}, { flushInitialActivity: false }, watch); - stopClarity(); - const sentEvents = stopWatching().sentEvents; - assert.isAbove(sentEvents.length, 0); - done(); - }); - - it("validates that config is reset to default on teardown", testAsync(async (done: DoneFn) => { - const mockSensitiveAttr: string = "mock-sensitive-attribute"; - - watch(); - const triggerAttributes = {}; - triggerAttributes[mockSensitiveAttr] = "1"; - await triggerMutationEventAndWaitForUpload(triggerAttributes); - - let events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.attributes[mockSensitiveAttr], "1"); - - // Sensitive attributes are masked - await restartClarity({ sensitiveAttributes: [mockSensitiveAttr] }); - await restartClarity(); - - watch(); - await triggerMutationEventAndWaitForUpload(triggerAttributes); - - events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.attributes[mockSensitiveAttr], "1"); - done(); - })); - -}); diff --git a/karma/tests/core/trigger.ts b/karma/tests/core/trigger.ts deleted file mode 100644 index 0f883642..00000000 --- a/karma/tests/core/trigger.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { stopClarity, triggerClarityAndWaitForUpload } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { testAsync } from "@karma/setup/testasync"; -import { getFullImpressionWatchResult } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Core: Trigger Tests", () => { - - beforeEach((done: DoneFn) => setupPage(done, { backgroundMode: true })); - afterEach(cleanupPage); - - it("validates that nothing is sent on teardown in background mode without trigger", (done: DoneFn) => { - stopClarity(); - const allSentEvents = getFullImpressionWatchResult().sentEvents; - assert.equal(allSentEvents.length, 0); - done(); - }); - - it("validates that upload queue is flushed when Clarity trigger is fired", testAsync(async (done: DoneFn) => { - let allSentEvents = getFullImpressionWatchResult().sentEvents; - assert.equal(allSentEvents.length, 0); - await triggerClarityAndWaitForUpload(); - allSentEvents = getFullImpressionWatchResult().sentEvents; - assert.isAbove(allSentEvents.length, 0); - done(); - })); - -}); diff --git a/karma/tests/core/upload.ts b/karma/tests/core/upload.ts deleted file mode 100644 index 1848fb4a..00000000 --- a/karma/tests/core/upload.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { UploadCallback } from "@clarity-types/core"; -import { Instrumentation } from "@clarity-types/instrumentation"; -import { Action } from "@clarity-types/layout"; -import { getActiveConfig, restartClarity, triggerMutationEvent, triggerMutationEventAndWaitForUpload } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -const InstrumentationEventName = "Instrumentation"; -const LayoutEventName = "Layout"; - -describe("Data Upload Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("validates that custom upload handler is invoked when passed through config", testAsync(async (done: DoneFn) => { - let customerUploadHandlerInvoked: boolean = false; - function customUploadHandler(payload: string, onSuccess: UploadCallback, onFailure?: UploadCallback): void { - customerUploadHandlerInvoked = true; - } - await restartClarity({ uploadHandler: customUploadHandler }); - - assert.equal(customerUploadHandlerInvoked, true); - done(); - })); - - it("validates that XhrError is logged for failed requests through the 'onFailure' upload callback", - testAsync(async (done: DoneFn) => { - let shouldMockFailure: boolean = false; - function customUploadHandler(payload: string, onSuccess: UploadCallback, onFailure?: UploadCallback): void { - if (shouldMockFailure) { - // Suppose XHR was opened and returned a 400 code - onFailure(400); - } - } - await restartClarity({ - instrument: true, - uploadHandler: customUploadHandler - }); - - shouldMockFailure = true; - - // Trigger event and wait for it to be processed by core, so that it doesn't pollute our watch - triggerMutationEvent(); - await waitFor(PubSubEvents.MUTATION); - - watch(); - jasmine.clock().tick(getActiveConfig().delay); - await waitFor(PubSubEvents.UPLOAD); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].type, InstrumentationEventName); - assert.equal(events[0].state.type, Instrumentation.XhrError); - assert.equal(events[0].state.requestStatus, 400); - done(); - }) - ); - - it("validates that Clarity tears down and logs instrumentation when total byte limit is exceeded", - testAsync(async (done: DoneFn) => { - let shouldMockSuccess: boolean = false; - function customUploadHandler(payload: string, onSuccess: UploadCallback, onFailure?: UploadCallback): void { - if (shouldMockSuccess) { - // Internal onSuccess callback is where clarity increments uploaded bytes and matches it against the configured limit - onSuccess(200); - } - } - await restartClarity({ - instrument: true, - totalLimit: 0, - uploadHandler: customUploadHandler - }); - - shouldMockSuccess = true; - - // Trigger event and wait for it to be processed by core, so that it doesn't pollute our watch - triggerMutationEvent(); - await waitFor(PubSubEvents.MUTATION); - - watch(); - jasmine.clock().tick(getActiveConfig().delay); - await waitFor(PubSubEvents.UPLOAD); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].state.type, Instrumentation.TotalByteLimitExceeded); - assert.equal(events[1].state.type, Instrumentation.Teardown); - done(); - }) - ); - - it("validates that dropped payloads are re-sent through the next request's 'onSuccess' callback", - testAsync(async (done: DoneFn) => { - const triggerKeyAttrName: string = "data-trigger-key"; - let shouldMockSuccess: boolean = false; - let shouldMockFailure: boolean = false; - function customUploadHandler(payload: string, onSuccess: UploadCallback, onFailure?: UploadCallback): void { - if (shouldMockSuccess) { - // Internal onSuccess callback is where clarity increments uploaded bytes and matches it against the configured limit - onSuccess(200); - } - if (shouldMockFailure) { - // Suppose XHR was opened and returned a 400 code - onFailure(400); - } - } - await restartClarity({ - instrument: true, - uploadHandler: customUploadHandler - }); - - // Part 1: Mock an XHR failure, so that trigger event 1 is stored for re-delivery and XHR failure instrumentation is logged - shouldMockFailure = true; - const firstTriggerAttrs = {}; - firstTriggerAttrs[triggerKeyAttrName] = "1"; - const firstTrigger = await triggerMutationEventAndWaitForUpload(firstTriggerAttrs); - - // Part 2: Successfully send second payload, which should trigger re-send of the first payload - shouldMockFailure = false; - shouldMockSuccess = true; - watch(); - const secondTriggerAttrs = {}; - secondTriggerAttrs[triggerKeyAttrName] = "2"; - const secondTrigger = await triggerMutationEventAndWaitForUpload(secondTriggerAttrs); - - // We expect to have 5 events: - // 1. XHR error: this event got generated immediately after trigger event 1 upload failed - // 2. Trigger event 2: Our manually-crafted second trigger, after which we waited for upload (#1 + #2 event batch) - // 3. Mutation perf - // 4. Trigger event 1: Upload from the line above, synchronously triggers a re-upload of event trigger 1 - // 5. Mutation perf - const sentEvents = stopWatching().sentEvents; - assert.equal(sentEvents.length, 5); - assert.equal(sentEvents[0].type, InstrumentationEventName); - assert.equal(sentEvents[0].state.type, Instrumentation.XhrError); - assert.equal(sentEvents[0].state.requestStatus, 400); - assert.equal(sentEvents[1].type, LayoutEventName); - assert.equal(sentEvents[1].state.action, Action.Insert); - assert.equal(sentEvents[1].state.attributes[triggerKeyAttrName], secondTrigger.attributes[triggerKeyAttrName].value); - assert.equal(sentEvents[2].type, InstrumentationEventName); - assert.equal(sentEvents[2].state.procedure, "Mutation"); - assert.equal(sentEvents[3].type, LayoutEventName); - assert.equal(sentEvents[3].state.action, Action.Insert); - assert.equal(sentEvents[3].state.attributes[triggerKeyAttrName], firstTrigger.attributes[triggerKeyAttrName].value); - assert.equal(sentEvents[4].type, InstrumentationEventName); - assert.equal(sentEvents[4].state.procedure, "Mutation"); - done(); - }) - ); - -}); diff --git a/karma/tests/instrumentation/activate.ts b/karma/tests/instrumentation/activate.ts deleted file mode 100644 index 189e99a2..00000000 --- a/karma/tests/instrumentation/activate.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Instrumentation } from "@clarity-types/instrumentation"; -import { restartClarity } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { testAsync } from "@karma/setup/testasync"; -import { getFullImpressionWatchResult } from "@karma/setup/watch"; -import { ClarityAttribute } from "@src/core"; -import { assert } from "chai"; - -const instrumentationEventName = "Instrumentation"; - -describe("Instrumentation: Activate Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("validates that missing feature event is sent when required feature is missing", testAsync(async (done: DoneFn) => { - - // Function.prototype.bind is a required API for Clarity to work - // Mocking a browser that doesn't support it by temporarily deleting it - const _bind = Function.prototype.bind; - Function.prototype.bind = undefined; - await restartClarity({ instrument: true }); - Function.prototype.bind = _bind; - - const events = getFullImpressionWatchResult().sentEvents; - assert.equal(events.length, 2); - assert.equal(events[0].type, instrumentationEventName); - assert.equal(events[0].state.type, Instrumentation.MissingFeature); - assert.equal(events[1].type, instrumentationEventName); - assert.equal(events[1].state.type, Instrumentation.Teardown); - done(); - })); - - it("validates that error during clarity activate is caught and logged correctly", testAsync(async (done: DoneFn) => { - const _worker = Worker; - const mockErrorText = "Mock error!"; - Worker = ((): void => { throw new Error(mockErrorText); }) as any; - await restartClarity({ instrument: true }); - Worker = _worker; - - const events = getFullImpressionWatchResult().sentEvents; - assert.equal(events.length, 2); - assert.equal(events[0].type, instrumentationEventName); - assert.equal(events[0].state.type, Instrumentation.ClarityActivateError); - assert.equal(events[0].state.error, mockErrorText); - assert.equal(events[1].type, instrumentationEventName); - assert.equal(events[1].state.type, Instrumentation.Teardown); - done(); - })); - - it("validates that Clarity logs instrumentation and tears down, when another instance of Clarity is running", - testAsync(async (done: DoneFn) => { - const mockExistingImpressionId = "MockExistingImpressionId"; - await restartClarity({ instrument: true }, null, () => { - document[ClarityAttribute] = mockExistingImpressionId; - }); - - const events = getFullImpressionWatchResult().sentEvents; - assert.equal(events.length, 2); - assert.equal(events[0].type, instrumentationEventName); - assert.equal(events[0].state.type, Instrumentation.ClarityDuplicated); - assert.equal(events[0].state.currentImpressionId, mockExistingImpressionId); - assert.equal(events[1].type, instrumentationEventName); - assert.equal(events[1].state.type, Instrumentation.Teardown); - done(); - }) - ); - -}); diff --git a/karma/tests/instrumentation/error.ts b/karma/tests/instrumentation/error.ts deleted file mode 100644 index 71bd0de8..00000000 --- a/karma/tests/instrumentation/error.ts +++ /dev/null @@ -1,101 +0,0 @@ -// @ts-ignore: 'merge' implicityly has 'any' type. There are no typings for 'merge' -import * as merge from "merge"; - -import { Instrumentation } from "@clarity-types/instrumentation"; -import { MockErrorInit, MockErrorMessage } from "@karma/setup/mocks/error"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Error Tests", () => { - - let jasmineErrorHandler: ErrorEventHandler = null; - - function preventDefault(e: Event): void { - e.preventDefault(); - } - - beforeEach((done: DoneFn) => { - setupPage(done, ({ instrument: true })); - jasmineErrorHandler = window.onerror; - window.onerror = null; - window.addEventListener("error", preventDefault); - }); - afterEach(() => { - window.removeEventListener("error", preventDefault); - window.onerror = jasmineErrorHandler; - cleanupPage(); - }); - - it("checks that a single error event is logged", (done: DoneFn) => { - watch(); - throwError(MockErrorMessage); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.type, Instrumentation.JsError); - assert.equal(events[0].state.message, MockErrorMessage); - assert.equal(events[0].state.source, MockErrorInit.filename); - assert.equal(events[0].state.lineno, MockErrorInit.lineno); - assert.equal(events[0].state.colno, MockErrorInit.colno); - done(); - }); - - it("checks empty message when not passed", (done: DoneFn) => { - watch(); - throwError(); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.type, Instrumentation.JsError); - assert.equal(events[0].state.message, "undefined"); - assert.equal(events[0].state.source, MockErrorInit.filename); - assert.equal(events[0].state.lineno, MockErrorInit.lineno); - assert.equal(events[0].state.colno, MockErrorInit.colno); - done(); - }); - - it("checks that multiple error events are logged", (done: DoneFn) => { - const secondMockErrorMessage = "SecondMockErrorMessage"; - watch(); - throwError(MockErrorMessage); - throwError(secondMockErrorMessage); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].state.type, Instrumentation.JsError); - assert.equal(events[0].state.message, MockErrorMessage); - assert.equal(events[1].state.type, Instrumentation.JsError); - assert.equal(events[1].state.message, secondMockErrorMessage); - done(); - }); - - it("checks that error event's message is logged, when error object is not provided", (done: DoneFn) => { - watch(); - throwError(null, { error: null }); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.type, Instrumentation.JsError); - assert.equal(events[0].state.message, MockErrorInit.message); - done(); - }); - - function throwError(errMessage?: string, errEventInit?: ErrorEventInit): void { - const errInit = merge(true, MockErrorInit, errEventInit); - const script = document.createElement("script"); - script.type = "text/javascript"; - script.innerText = ` - (function() { - var errInit = ${JSON.stringify(errInit)}; - if (!("error" in errInit)) { - errInit.error = new Error("${errMessage}"); - } - var errEvt = new ErrorEvent("error", errInit); - window.dispatchEvent(errEvt); - }()); - `; - document.body.appendChild(script); - } - -}); diff --git a/karma/tests/instrumentation/perf.ts b/karma/tests/instrumentation/perf.ts deleted file mode 100644 index 039898c2..00000000 --- a/karma/tests/instrumentation/perf.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { IEvent } from "@clarity-types/core"; -import { Instrumentation } from "@clarity-types/instrumentation"; -import { restartClarity } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { getFullImpressionWatchResult, stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -const instrumentationEventName = "Instrumentation"; - -describe("Instrumentation: Performance Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("validates that discover performance instrumentation is logged", testAsync(async (done: DoneFn) => { - await restartClarity({ instrument: true }); - - const sentEvents = getFullImpressionWatchResult().sentEvents; - let discoverPerfEvent: IEvent = null; - for (let i = 0; i < sentEvents.length; i++) { - const event: IEvent = sentEvents[i]; - if ( - event.type === instrumentationEventName - && event.state.type === Instrumentation.Performance - && event.state.procedure === "Discover" - ) { - discoverPerfEvent = event; - } - } - - assert.notEqual(discoverPerfEvent, null); - assert.isNumber(discoverPerfEvent.state.duration); - assert.isAbove(discoverPerfEvent.state.nodeCount, 0); - done(); - })); - - it("validates that mutation performance instrumentation is logged", testAsync(async (done: DoneFn) => { - await restartClarity({ instrument: true }); - - watch(); - let div = document.createElement("div"); - document.body.appendChild(div); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].type, "Layout"); - assert.equal(events[1].type, instrumentationEventName); - assert.equal(events[1].state.type, Instrumentation.Performance); - assert.equal(events[1].state.procedure, "Mutation"); - assert.equal(events[1].state.mutationCount, 1); - assert.equal(events[1].state.mutationSequence, 0); - assert.isNumber(events[1].state.stateGenDuration); - assert.equal(events[1].state.summaryCounts.inserts, 1); - assert.equal(events[1].state.summaryCounts.moves, 0); - assert.equal(events[1].state.summaryCounts.updates, 0); - assert.equal(events[1].state.summaryCounts.removes, 0); - - done(); - })); -}); diff --git a/karma/tests/instrumentation/trigger.ts b/karma/tests/instrumentation/trigger.ts deleted file mode 100644 index 2fb27140..00000000 --- a/karma/tests/instrumentation/trigger.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Instrumentation } from "@clarity-types/instrumentation"; -import { restartClarity, triggerClarity } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -const instrumentationEventName = "Instrumentation"; - -describe("Instrumentation: Trigger Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("validates that trigger instrumentation is logged when Clarity trigger is fired", testAsync(async (done: DoneFn) => { - const triggerKey = "Test Trigger"; - await restartClarity({ instrument: true, backgroundMode: true }); - - watch(); - triggerClarity(triggerKey); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].type, instrumentationEventName); - assert.equal(events[0].state.type, Instrumentation.Trigger); - assert.equal(events[0].state.key, triggerKey); - done(); - })); -}); diff --git a/karma/tests/layout/cssrules.ts b/karma/tests/layout/cssrules.ts deleted file mode 100644 index 1de18554..00000000 --- a/karma/tests/layout/cssrules.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Action } from "@clarity-types/layout"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Layout: CSS Rules Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that we capture cssRule modifications via javascript when no innerText found", testAsync(async (done: DoneFn) => { - watch(); - - // Add a style tag and later modify styles using javascript - let dom = document.getElementById("clarity"); - let style = document.createElement("style"); - dom.appendChild(style); - let stylesheet = style.sheet as CSSStyleSheet; - stylesheet.insertRule("body { color: red; }"); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - - // Assert that style state has css rules and that style's child text node is ignored - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(!!events[0].state.cssRules, true); - assert.equal(events[0].state.cssRules.length, 1); - assert.equal(events[0].state.cssRules[0].indexOf("red") > 0, true); - done(); - })); - -}); diff --git a/karma/tests/layout/input.ts b/karma/tests/layout/input.ts deleted file mode 100644 index 1d8def36..00000000 --- a/karma/tests/layout/input.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Action, Source } from "@clarity-types/layout"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { mask } from "@src/utils"; -import { assert } from "chai"; - -describe("Layout: Input Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that input change capturing works on inserted element", testAsync(async (done: DoneFn) => { - let newValueString = "new value"; - let input = document.createElement("input"); - document.body.appendChild(input); - await waitFor(PubSubEvents.MUTATION); - - watch(); - input.value = newValueString; - - // Programmatic value change doesn't trigger "onchange" event, so we need to trigger it manually - let onChangeEvent = document.createEvent("HTMLEvents"); - onChangeEvent.initEvent("change", false, true); - input.dispatchEvent(onChangeEvent); - await waitFor(PubSubEvents.CHANGE); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Update); - assert.equal(events[0].state.source, Source.Input); - assert.equal(events[0].state.value, mask(newValueString)); - done(); - })); - - it("checks that input change capturing works on inserted textarea element", testAsync(async (done: DoneFn) => { - let newValueString = "new value"; - let textarea = document.createElement("textarea"); - document.body.appendChild(textarea); - await waitFor(PubSubEvents.MUTATION); - - watch(); - textarea.value = newValueString; - - // Programmatic value change doesn't trigger "onchange" event, so we need to trigger it manually - let onInputEvent = document.createEvent("HTMLEvents"); - onInputEvent.initEvent("input", false, true); - textarea.dispatchEvent(onInputEvent); - await waitFor(PubSubEvents.INPUT); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Update); - assert.equal(events[0].state.source, Source.Input); - assert.equal(events[0].state.value, mask(newValueString)); - done(); - })); - -}); diff --git a/karma/tests/layout/mutation.ts b/karma/tests/layout/mutation.ts deleted file mode 100644 index 1a780e24..00000000 --- a/karma/tests/layout/mutation.ts +++ /dev/null @@ -1,547 +0,0 @@ -import { IEvent } from "@clarity-types/core"; -import { Action } from "@clarity-types/layout"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { NodeIndex, Tags } from "@src/plugins/layout/states/generic"; -import { assert } from "chai"; - -describe("Layout: Mutation Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that dom additions are captured by clarity", testAsync(async (done: DoneFn) => { - watch(); - let div = document.createElement("div"); - let span = document.createElement("span"); - span.innerHTML = "Clarity"; - div.appendChild(span); - document.body.insertBefore(div, document.body.firstChild); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 3); - assert.equal(events[0].state.tag, "DIV"); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[1].state.tag, "SPAN"); - assert.equal(events[2].state.tag, "*TXT*"); - done(); - })); - - it("checks that dom removals are captured by clarity", testAsync(async (done: DoneFn) => { - watch(); - let dom = document.getElementById("clarity"); - dom.parentNode.removeChild(dom); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.tag, "DIV"); - assert.equal(events[0].state.action, Action.Remove); - done(); - })); - - it("checks that dom moves are captured correctly by clarity", testAsync(async (done: DoneFn) => { - watch(); - let dom = document.getElementById("clarity"); - let backup = document.getElementById("backup"); - backup.appendChild(dom); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Move); - done(); - })); - - it("checks that insertBefore works correctly", testAsync(async (done: DoneFn) => { - watch(); - - // Insert a node before an existing node - let dom = document.getElementById("clarity"); - let backup = document.getElementById("backup"); - let domIndex = dom[NodeIndex]; - let firstChildIndex = dom.firstChild[NodeIndex]; - dom.insertBefore(backup, dom.firstChild); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Move); - assert.equal(events[0].state.parent, domIndex); - assert.equal(events[0].state.next, firstChildIndex); - done(); - })); - - it("checks that moving two known nodes to a new location such that they are siblings works correctly", - testAsync(async (done: DoneFn) => { - watch(); - - // Move multiple nodes from one parent to another - let dom = document.getElementById("clarity"); - let backup = document.getElementById("backup"); - let span = document.createElement("span"); - dom.parentElement.appendChild(span); - span.appendChild(dom); - span.appendChild(backup); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - - assert.equal(events.length, 3); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, "SPAN"); - - assert.equal(events[1].state.action, Action.Move); - assert.equal(events[1].state.index, dom[NodeIndex]); - assert.equal(events[1].state.next, backup[NodeIndex]); - - assert.equal(events[2].state.action, Action.Move); - assert.equal(events[2].state.index, backup[NodeIndex]); - assert.equal(events[2].state.next, null); - done(); - }) - ); - - it("checks dom changes are captured accurately when multiple siblings are moved to another parent", testAsync(async (done: DoneFn) => { - watch(); - - // Move multiple nodes from one parent to another - let dom = document.getElementById("clarity"); - let backup = document.getElementById("backup"); - let backupIndex = backup[NodeIndex]; - let childrenCount = dom.childNodes.length; - while (dom.childNodes.length > 0) { - backup.appendChild(dom.childNodes[0]); - } - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, childrenCount); - assert.equal(events[0].state.action, Action.Move); - assert.equal(events[0].state.parent, backupIndex); - assert.equal(events[4].state.parent, backupIndex); - assert.equal(events[4].state.next, events[5].state.index); - done(); - })); - - it("checks that insertion of multiple nodes in the same mutation record is handled correctly", testAsync(async (done: DoneFn) => { - watch(); - let df = document.createDocumentFragment(); - let n1 = document.createElement("div"); - let n2 = document.createElement("span"); - let n3 = document.createElement("a"); - let backup = document.getElementById("backup"); - let backupPrevious = backup.previousSibling; - df.appendChild(n1); - df.appendChild(n2); - df.appendChild(n3); - backup.parentElement.insertBefore(df, backup); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - - assert.equal(events.length, 3); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, "DIV"); - assert.equal(events[0].state.previous, backupPrevious[NodeIndex]); - assert.equal(events[0].state.next, n2[NodeIndex]); - - // Check SPAN insert event - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.tag, "SPAN"); - assert.equal(events[1].state.next, n3[NodeIndex]); - - // Check A insert event - assert.equal(events[2].state.action, Action.Insert); - assert.equal(events[2].state.tag, "A"); - assert.equal(events[2].state.next, backup[NodeIndex]); - done(); - })); - - it("checks that removal of multiple nodes in the same mutation record is handled correctly", testAsync(async (done: DoneFn) => { - watch(); - let dom = document.getElementById("clarity"); - let children = []; - let childIndices = []; - for (let i = 0; i < dom.childNodes.length; i++) { - let child = dom.childNodes[i]; - let index = child[NodeIndex]; - children.push(child); - childIndices[index] = true; - } - - // Remove all children - dom.innerHTML = ""; - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - - // Make sure that there is a remove event for every child - for (let i = 0; i < events.length; i++) { - let index = events[i].state.index; - assert.equal(events[i].state.action, Action.Remove); - assert.equal(childIndices[index], true); - delete childIndices[index]; - } - - // Make sure that clarity index is cleared from all removed nodes - for (let i = 0; i < children.length; i++) { - assert.equal(NodeIndex in children[i], false); - } - done(); - })); - - // Nodes that are inserted and then removed in the same mutation don't produce any events and their mutations are ignored - // However, it's possible that some other observed node can be appended to the ignored node and then get removed from the - // DOM as a part of the ignored node's subtree. This test makes sure that removing observed node this way is captured correctly. - it("checks that removal of a known node through a subtree of its ignored parent is handled correctly", - testAsync(async (done: DoneFn) => { - watch(); - let clarityNode = document.getElementById("clarity"); - let backupNode = document.getElementById("backup"); - let backupNodeIndex = backupNode[NodeIndex]; - let tempNode = document.createElement("div"); - clarityNode.appendChild(tempNode); - tempNode.appendChild(backupNode); - clarityNode.removeChild(tempNode); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Remove); - assert.equal(events[0].state.index, backupNodeIndex); - - // Make sure that clarity index is cleared from all removed nodes - assert.equal(NodeIndex in tempNode, false); - assert.equal(NodeIndex in backupNode, false); - done(); - }) - ); - - it("checks that dom addition with immediate move ignores the 'Move' action", testAsync(async (done: DoneFn) => { - watch(); - - // Add a node to the document - let clarity = document.getElementById("clarity"); - let div = document.createElement("div"); - document.body.appendChild(div); - clarity.appendChild(div); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, "DIV"); - assert.equal(events[0].state.parent, clarity[NodeIndex]); - done(); - })); - - it("checks that dom addition with the follow-up attribute change captures the 'Update' action", testAsync(async (done: DoneFn) => { - watch(); - let div = document.createElement("div"); - document.body.appendChild(div); - await waitFor(PubSubEvents.MUTATION); - - div.setAttribute("data-clarity", "test"); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, "DIV"); - assert.equal(events[1].state.action, Action.Update); - done(); - })); - - it("checks that dom addition with immediate attribute change ignores the 'Update' action", testAsync(async (done: DoneFn) => { - watch(); - let div = document.createElement("div"); - document.body.appendChild(div); - div.setAttribute("data-clarity", "test"); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, "DIV"); - done(); - })); - - it("checks that nodes that are added and removed in the same mutation don't create index gaps in event logs ", - testAsync(async (done: DoneFn) => { - watch(); - let div1 = document.createElement("div"); - let div2 = document.createElement("div"); - let div3 = document.createElement("div"); - document.body.appendChild(div1); - document.body.appendChild(div3); - document.body.appendChild(div2); - document.body.removeChild(div3); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.index, div1[NodeIndex]); - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.index, div2[NodeIndex]); - assert.equal(div1[NodeIndex], div2[NodeIndex] - 1); - done(); - }) - ); - - it("checks that we do not instrument disconnected dom tree", testAsync(async (done: DoneFn) => { - watch(); - let div = document.createElement("div"); - document.body.appendChild(div); - document.body.removeChild(div); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - - // Prove that we didn't send any extra instrumentation back for no-op mutation - assert.equal(events.length, 0); - - // Make sure that clarity index is cleared from all removed nodes - assert.equal(NodeIndex in div, false); - done(); - })); - - it("checks that we do not instrument child nodes within disconnected dom tree", testAsync(async (done: DoneFn) => { - watch(); - let div = document.createElement("div"); - let span = document.createElement("span"); - document.body.appendChild(div); - div.appendChild(span); - document.body.removeChild(div); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - - // Prove that we didn't send any extra instrumentation back for no-op mutation - assert.equal(events.length, 0); - - // Make sure that clarity index is cleared from all removed nodes - assert.equal(NodeIndex in div, false); - assert.equal(NodeIndex in span, false); - done(); - })); - - it("checks that we do not instrument child nodes within a previously observed disconnected dom tree", - testAsync(async (done: DoneFn) => { - watch(); - let clarityDiv = document.getElementById("clarity"); - let span = document.createElement("span"); - let clarityDivIndex = clarityDiv[NodeIndex]; - clarityDiv.appendChild(span); - clarityDiv.parentElement.removeChild(clarityDiv); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - - // Prove that we didn't send any extra instrumentation back for no-op mutation - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Remove); - assert.equal(events[0].state.index, clarityDivIndex); - - // Make sure that clarity index is cleared from all removed nodes - assert.equal(NodeIndex in clarityDiv, false); - assert.equal(NodeIndex in span, false); - done(); - }) - ); - - it("checks that we do not instrument inserted nodes twice", testAsync(async (done: DoneFn) => { - watch(); - - // Edge case scenario for the test: - // 1. Node n1 is added to the page - // 2. Immediately node n2 is appended to n1 - // They both show up as insertions in the same batch of mutations - // When we serialize n1, we will discover its children and serialize n2 as well - // Make sure we don't serialize n2 again when we move on to n2 insertion mutation record - let n1 = document.createElement("div"); - let n2 = document.createElement("span"); - let bodyIndex = document.body[NodeIndex]; - document.body.appendChild(n1); - n1.appendChild(n2); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - - // Check DIV insert event - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.parent, bodyIndex); - assert.equal(events[0].state.tag, "DIV"); - - // Check SPAN insert event - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.parent, n1[NodeIndex]); - assert.equal(events[1].state.tag, "SPAN"); - done(); - })); - - it("checks that all kinds of mutations within the same batch have the same mutation sequence", testAsync(async (done: DoneFn) => { - watch(); - let divOne = document.createElement("div"); - let clarityDiv = document.getElementById("clarity"); - let backup = document.getElementById("backup"); - document.body.appendChild(divOne); // Insert - clarityDiv.firstElementChild.id = "updatedId"; // Update - divOne.appendChild(clarityDiv); // Move - backup.parentElement.removeChild(backup); // Remove - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 4); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[1].state.action, Action.Move); - assert.equal(events[2].state.action, Action.Update); - assert.equal(events[3].state.action, Action.Remove); - - // Make sure all events have the same mutation sequence - let mutationSequence = events[0].state.mutationSequence; - assert.isTrue(mutationSequence >= 0); - assert.equal(events[0].state.mutationSequence, mutationSequence); - assert.equal(events[1].state.mutationSequence, mutationSequence); - assert.equal(events[2].state.mutationSequence, mutationSequence); - assert.equal(events[3].state.mutationSequence, mutationSequence); - done(); - })); - - it("checks that mutation sequence number is incremented between mutation callbacks", testAsync(async (done: DoneFn) => { - watch(); - let divOne = document.createElement("div"); - let divTwo = document.createElement("div"); - document.body.appendChild(divOne); - await waitFor(PubSubEvents.MUTATION); - - document.body.appendChild(divTwo); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].state.mutationSequence, 0); - assert.equal(events[1].state.mutationSequence, 1); - done(); - })); - - it("checks that script element and its text are ignored", testAsync(async (done: DoneFn) => { - watch(); - let script = document.createElement("script"); - script.innerText = "/*some javascriptcode*/"; - document.body.appendChild(script); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, Tags.Ignore); - assert.equal(events[0].state.nodeType, Node.ELEMENT_NODE); - - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.tag, Tags.Ignore); - assert.equal(events[1].state.nodeType, Node.TEXT_NODE); - done(); - })); - - it("checks that comment node is ignored", testAsync(async (done: DoneFn) => { - watch(); - let comment = document.createComment("some explanation"); - document.body.appendChild(comment); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, Tags.Ignore); - assert.equal(events[0].state.nodeType, Node.COMMENT_NODE); - done(); - })); - - it("checks that meta element is ignored", testAsync(async (done: DoneFn) => { - watch(); - let meta = document.createElement("meta"); - document.body.appendChild(meta); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, Tags.Ignore); - assert.equal(events[0].state.nodeType, Node.ELEMENT_NODE); - done(); - })); - - // This test is related to a specific MutationObserver behavior in Internet Explorer for this scenario: - // 1. Append script 'myscript' to the page - // 2. 'myscript' executes and appends 'mydiv' div to the page - // 3. Inspect MutationObserver callback - // In Chrome: - // MutationObserver will invoke a callback and show 2 mutation records: - // (1) script added - // (2)
added - // In IE: - // (1) MutationObserver will invoke first callback with 1 mutation record:
added - // (2) MutationObserver will invoke second callback with 1 mutation record: script added - // The problem with this behavior in IE is that during the first MutationObserver's callback, script element - // can already be observed in the DOM, even though its mutation is not reported in the callback. - // As a result, after processing (1), DOM and ShadowDOM states are not consistent until (2) is processed. - // This breaks functionality, because after (1) we determine that ShadowDOM arrived to the inconsistent - // state and stop processing mutations. - // Solution: - // Wait for 2 consequtive mutations that bring ShadowDOM to the inconsistent state before disabling mutation processing. - it("checks that inserting script, which inserts an element, works correctly", testAsync(async (done: DoneFn) => { - watch(); - let events: IEvent[] = []; - let script = document.createElement("script"); - script.type = "text/javascript"; - script.innerHTML = `var div=document.createElement("div");div.id="newdiv";document.body.appendChild(div);`; - document.body.appendChild(script); - await waitFor(PubSubEvents.MUTATION); - - let newEvents = stopWatching().coreEvents; - events = events.concat(newEvents); - - if (events.length === 1) { - watch(); - await waitFor(PubSubEvents.MUTATION); - - newEvents = stopWatching().coreEvents; - events = events.concat(newEvents); - watch(); - - // Add another mutation to ensure that we continue processing mutations - document.body.appendChild(document.createElement("span")); - await waitFor(PubSubEvents.MUTATION); - - newEvents = stopWatching().coreEvents; - assert.equal(events.length, 4); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, "DIV"); - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.tag, Tags.Ignore); - assert.equal(events[2].state.action, Action.Insert); - assert.equal(events[2].state.tag, Tags.Ignore); - assert.equal(events[3].state.action, Action.Insert); - assert.equal(events[3].state.tag, "SPAN"); - done(); - } else { - // Non-IE path: Both div and script are reported in the first callback - assert.equal(events.length, 3); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, Tags.Ignore); - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.tag, Tags.Ignore); - assert.equal(events[2].state.action, Action.Insert); - assert.equal(events[2].state.tag, "DIV"); - done(); - } - })); -}); diff --git a/karma/tests/layout/privacy.ts b/karma/tests/layout/privacy.ts deleted file mode 100644 index 5406ea34..00000000 --- a/karma/tests/layout/privacy.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { Action, Source } from "@clarity-types/layout"; -import { restartClarity } from "@karma/setup/clarity"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { mask } from "@src/utils"; -import { assert } from "chai"; - -describe("Layout: Privacy Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that default sensitive attributes are masked", testAsync(async (done: DoneFn) => { - const sensitiveAttributeName = "placeholder"; - watch(); - let value = "value"; - let maskedValue = mask(value); - let div = document.createElement("div"); - div.setAttribute(sensitiveAttributeName, value); - document.body.appendChild(div); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, div.tagName); - assert.equal(events[0].state.attributes[sensitiveAttributeName], maskedValue); - done(); - })); - - it("checks that configurable sensitive attributes are masked", testAsync(async (done: DoneFn) => { - const sensitiveAttributeName = "data-sensitive-attribute"; - await restartClarity({ sensitiveAttributes: [sensitiveAttributeName] }); - - watch(); - let value = "value"; - let maskedValue = mask(value); - let div = document.createElement("div"); - div.setAttribute(sensitiveAttributeName, value); - document.body.appendChild(div); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, div.tagName); - assert.equal(events[0].state.attributes[sensitiveAttributeName], maskedValue); - done(); - })); - - it("checks that input value is masked on insert", testAsync(async (done: DoneFn) => { - watch(); - let input = document.createElement("input"); - let valueString = "value"; - let maskedValueString = mask(valueString); - input.setAttribute("value", valueString); - document.body.appendChild(input); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.tag, "INPUT"); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.value, maskedValueString); - done(); - })); - - it("checks that input value is masked on input update", testAsync(async (done: DoneFn) => { - let newValueString = "new value"; - let maskedValueString = mask(newValueString); - let input = document.createElement("input"); - document.body.appendChild(input); - await waitFor(PubSubEvents.MUTATION); - - watch(); - input.value = newValueString; - - // Programmatic value change doesn't trigger "onchange" event, so we need to trigger it manually - let onChangeEvent = document.createEvent("HTMLEvents"); - onChangeEvent.initEvent("change", false, true); - input.dispatchEvent(onChangeEvent); - await waitFor(PubSubEvents.CHANGE); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Update); - assert.equal(events[0].state.source, Source.Input); - assert.equal(events[0].state.value, maskedValueString); - done(); - })); - -}); diff --git a/karma/tests/layout/scroll.ts b/karma/tests/layout/scroll.ts deleted file mode 100644 index 5ff4369e..00000000 --- a/karma/tests/layout/scroll.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { Action, Source } from "@clarity-types/layout"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { assert } from "chai"; - -describe("Layout: Scroll Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that scroll capturing works on inserted element", testAsync(async (done: DoneFn) => { - - // Add a scrollable DIV - let outerDiv = document.createElement("div"); - let innerDiv = document.createElement("div"); - outerDiv.style.overflowY = "auto"; - outerDiv.style.width = "200px"; - outerDiv.style.maxHeight = "100px"; - innerDiv.style.height = "300px"; - outerDiv.appendChild(innerDiv); - document.body.appendChild(outerDiv); - await waitFor(PubSubEvents.MUTATION); - - watch(); - - // Trigger scroll - outerDiv.scrollTop = 100; - await waitFor(PubSubEvents.SCROLL); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Update); - assert.equal(events[0].state.source, Source.Scroll); - done(); - })); - - it("checks that scroll capturing works on overflow hidden element after a mutation update", testAsync(async (done: DoneFn) => { - - // Add a scrollable DIV - let outerDiv = document.createElement("div"); - let innerDiv = document.createElement("div"); - outerDiv.style.overflowY = "hidden"; - outerDiv.style.width = "200px"; - outerDiv.style.maxHeight = "100px"; - innerDiv.style.height = "300px"; - outerDiv.appendChild(innerDiv); - document.body.appendChild(outerDiv); - await waitFor(PubSubEvents.MUTATION); - - // Force a mutation to ensure that layout updates also capture scroll position - outerDiv.setAttribute("data-attribute", "1"); - await waitFor(PubSubEvents.MUTATION); - - watch(); - - // Trigger scroll after a mutation update - outerDiv.scrollTop = 100; - await waitFor(PubSubEvents.SCROLL); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Update); - assert.equal(events[0].state.source, Source.Scroll); - done(); - })); - - it("checks that scroll capturing works on element that enables scrolling after a mutation update", testAsync(async (done: DoneFn) => { - - // Add a scrollable DIV - let outerDiv = document.createElement("div"); - let innerDiv = document.createElement("div"); - outerDiv.style.overflowY = "visible"; - outerDiv.style.width = "200px"; - innerDiv.style.height = "300px"; - outerDiv.appendChild(innerDiv); - document.body.appendChild(outerDiv); - await waitFor(PubSubEvents.MUTATION); - - // Make the element scrollable - outerDiv.style.maxHeight = "100px"; - outerDiv.style.overflowY = "hidden"; - - // Force a mutation to ensure that layout updates also capture scroll position - outerDiv.setAttribute("data-attribute", "1"); - await waitFor(PubSubEvents.MUTATION); - - watch(); - - // Trigger scroll after a mutation update - outerDiv.scrollTop = 100; - await waitFor(PubSubEvents.SCROLL); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.action, Action.Update); - assert.equal(events[0].state.source, Source.Scroll); - done(); - })); - -}); diff --git a/karma/tests/layout/unmask.ts b/karma/tests/layout/unmask.ts deleted file mode 100644 index 75658426..00000000 --- a/karma/tests/layout/unmask.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Action } from "@clarity-types/layout"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { UnmaskAttribute } from "@src/plugins/layout/nodeinfo"; -import { assert } from "chai"; - -describe("Layout: Unmasking Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that child value is unmasked when parent unmask attribute is applied", - testAsync(async (done: DoneFn) => { - watch(); - let valueString = "value"; - let input = document.createElement("input"); - let child = document.createTextNode(valueString); - input.setAttribute(UnmaskAttribute, "true"); - input.appendChild(child); - document.body.appendChild(input); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 2); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, input.tagName); - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.tag, "*TXT*"); - assert.equal(events[1].state.content, valueString); - done(); - }) - ); - - it(`checks that node value is unmasked when skip-parent unmask attribute is applied`, - testAsync(async (done: DoneFn) => { - watch(); - let valueString = "value"; - let skipParent = document.createElement("div"); - let parent = document.createElement("div"); - let child = document.createTextNode(valueString); - parent.appendChild(child); - skipParent.appendChild(parent); - skipParent.setAttribute(UnmaskAttribute, "true"); - document.body.appendChild(skipParent); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 3); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.tag, skipParent.tagName); - assert.equal(events[1].state.action, Action.Insert); - assert.equal(events[1].state.tag, parent.tagName); - assert.equal(events[2].state.action, Action.Insert); - assert.equal(events[2].state.tag, "*TXT*"); - assert.equal(events[2].state.content, valueString); - done(); - }) - ); - - it("checks that input value is unmasked on insert if unmask attribute is applied", - testAsync(async (done: DoneFn) => { - watch(); - let input = document.createElement("input"); - let valueString = "value"; - input.setAttribute("value", valueString); - input.setAttribute(UnmaskAttribute, "true"); - document.body.appendChild(input); - await waitFor(PubSubEvents.MUTATION); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 1); - assert.equal(events[0].state.tag, "INPUT"); - assert.equal(events[0].state.action, Action.Insert); - assert.equal(events[0].state.value, valueString); - done(); - }) - ); - -}); diff --git a/karma/tests/performance.ts b/karma/tests/performance.ts deleted file mode 100644 index 4905785a..00000000 --- a/karma/tests/performance.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { IPerformanceResourceTimingState, IPerformanceTiming } from "@clarity-types/performance"; -import { restartClarity } from "@karma/setup/clarity"; -import { createMockPerformanceResourceTimings, IMockPerformance } from "@karma/setup/mocks/performance"; -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { testAsync } from "@karma/setup/testasync"; -import { filterEventsByType, getFullImpressionWatchResult, stopWatching, watch } from "@karma/setup/watch"; -import { ResourceTimingEventType } from "@src/plugins/performance"; -import { assert } from "chai"; - -let resourceTimingEventName = "ResourceTiming"; -let stateErrorEventName = "PerformanceStateError"; -let navigationTimingEventName = "NavigationTiming"; - -declare const performance: IMockPerformance; - -describe("Performance Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("checks that w3c performance timing is logged by clarity", (done: DoneFn) => { - const sentEvents = getFullImpressionWatchResult().sentEvents; - const navTimingEvents = filterEventsByType(sentEvents, navigationTimingEventName); - assert.equal(navTimingEvents.length, 1); - - const eventTiming: IPerformanceTiming = navTimingEvents[0].state.timing; - assert.equal(eventTiming.connectStart, formatTiming(performance.timing.connectStart)); - assert.equal(eventTiming.connectEnd, formatTiming(performance.timing.connectEnd)); - done(); - }); - - it("checks that network resource timings are logged by clarity", (done: DoneFn) => { - watch(); - const resourceTiming = createMockPerformanceResourceTimings()[0]; - performance.addEntry(resourceTiming); - triggerNextPerformancePoll(); - - const events = filterEventsByType(stopWatching().coreEvents, resourceTimingEventName); - assert.equal(events.length, 1); - assert.equal(events[0].type, ResourceTimingEventType); - - // Check sample string value and sample numeric value - const eventState: IPerformanceResourceTimingState = events[0].state; - assert.equal(eventState.name, resourceTiming.name); - assert.equal(eventState.connectStart, Math.round(resourceTiming.connectStart)); - done(); - }); - - it("checks that error is logged when entries are cleared", (done: DoneFn) => { - watch(); - const resourceTiming = createMockPerformanceResourceTimings()[0]; - performance.addEntry(resourceTiming); - triggerNextPerformancePoll(); - - let events = filterEventsByType(stopWatching().coreEvents, resourceTimingEventName); - assert.equal(events.length, 1); - - watch(); - performance.clearResourceTimings(); - triggerNextPerformancePoll(); - - events = filterEventsByType(stopWatching().coreEvents, stateErrorEventName); - assert.equal(events.length, 1); - done(); - }); - - // "Incomplete" entries are entries for resources that are not finished loading yet - it("checks that incomplete entries are not logged initially, but then revisited", (done: DoneFn) => { - let resourceTimings = createMockPerformanceResourceTimings(); - const completeEntry = resourceTimings[0]; - const incompleteEntry = resourceTimings[1]; - const _responseEnd = incompleteEntry.responseEnd; - incompleteEntry.responseEnd = 0; - - watch(); - performance.addEntry(completeEntry); - performance.addEntry(incompleteEntry); - triggerNextPerformancePoll(); - - let events = filterEventsByType(stopWatching().coreEvents, resourceTimingEventName); - assert.equal(events.length, 1); - assert.equal(events[0].type, ResourceTimingEventType); - assert.equal(events[0].state.name, completeEntry.name); - - // Adjust the entry to have a valid response end time and wait for snapshot to propagate - watch(); - incompleteEntry.responseEnd = _responseEnd; - triggerNextPerformancePoll(); - - events = filterEventsByType(stopWatching().coreEvents, resourceTimingEventName); - assert.equal(events.length, 1); - assert.equal(events[0].type, ResourceTimingEventType); - assert.equal(events[0].state.name, incompleteEntry.name); - assert.equal(events[0].state.responseEnd, Math.round(_responseEnd)); - done(); - }); - - it("checks that network resource timings blacklisted in config are ignored", testAsync(async (done: DoneFn) => { - const blacklistedUrl = "https://www.blacklisted.url"; - const resourceTiming = createMockPerformanceResourceTimings()[0]; - await restartClarity({ urlBlacklist: [blacklistedUrl] }); - - watch(); - resourceTiming.name = getFullUrl(blacklistedUrl); - performance.addEntry(resourceTiming); - triggerNextPerformancePoll(); - - const events = filterEventsByType(stopWatching().coreEvents, resourceTimingEventName); - assert.equal(events.length, 0); - done(); - })); - - function formatTiming(value: number): number { - return value === 0 ? 0 : Math.round(value - performance.timing.navigationStart); - } - - function getFullUrl(partialUrl: string): string { - const a = document.createElement("a"); - a.href = partialUrl; - return a.href; - } - - function triggerNextPerformancePoll(): void { - jasmine.clock().tick(100000); - } -}); diff --git a/karma/tests/pointer.ts b/karma/tests/pointer.ts deleted file mode 100644 index 203e8cc2..00000000 --- a/karma/tests/pointer.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { cleanupPage, setupPage } from "@karma/setup/page"; -import { PubSubEvents, waitFor } from "@karma/setup/pubsub"; -import { testAsync } from "@karma/setup/testasync"; -import { stopWatching, watch } from "@karma/setup/watch"; -import { getTimestamp } from "@src/core"; -import { assert } from "chai"; - -let distanceThreshold = 20; -let timeThreshold = 500; - -describe("Pointer Tests", () => { - - beforeEach(setupPage); - afterEach(cleanupPage); - - it("validates that mouse events are processed by clarity", testAsync(async (done: DoneFn) => { - watch(); - - // Trigger mousemove events followed by a click event - let dom = document.getElementById("clarity"); - let x = 250; - let xDelta = (distanceThreshold + 1); - triggerMouseEvent(dom, "mousemove", x, 100); - triggerMouseEvent(dom, "mousemove", x + xDelta, 100); - triggerMouseEvent(dom, "mousemove", x + (xDelta * 2), 100); - triggerMouseEvent(dom, "click", 260, 100); - await waitFor(PubSubEvents.CLICK); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 4); - assert.equal(events[0].state.event, "mousemove"); - assert.equal(events[0].state.x, x); - assert.equal(events[1].state.event, "mousemove"); - assert.equal(events[1].state.x, x + xDelta); - assert.equal(events[2].state.event, "mousemove"); - assert.equal(events[2].state.x, x + (xDelta * 2)); - assert.equal(events[3].state.event, "click"); - done(); - })); - - // Make sure that we don't record mouse events that are too close to each other - it("validates that mouse events are throttled by distance", testAsync(async (done: DoneFn) => { - watch(); - - // Trigger mousemove events followed by a click event - let dom = document.getElementById("clarity"); - let x = 250; - let xDelta = Math.ceil(distanceThreshold / 2) + 1; - triggerMouseEvent(dom, "mousemove", x, 100); - triggerMouseEvent(dom, "mousemove", x + xDelta, 100); - triggerMouseEvent(dom, "mousemove", x + (xDelta * 2), 100); - triggerMouseEvent(dom, "click", 260, 100); - await waitFor(PubSubEvents.CLICK); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 3); - assert.equal(events[0].state.event, "mousemove"); - assert.equal(events[0].state.x, x); - assert.equal(events[1].state.event, "mousemove"); - assert.equal(events[1].state.x, x + (xDelta * 2)); - assert.equal(events[2].state.event, "click"); - done(); - })); - - // Make sure that we don't record mouse events that are too close to each other - it("validates that mouse events are throttled by time", testAsync(async (done: DoneFn) => { - watch(); - - // Trigger mousemove events followed by a click event - let dom = document.getElementById("clarity"); - let x = 250; - triggerMouseEvent(dom, "mousemove", x, 100); - triggerMouseEvent(dom, "mousemove", x, 100); - - let thresholdTs = getTimestamp(true) + (timeThreshold * 2); - while (getTimestamp(true) < thresholdTs) { - // Wait for time threadhold to expire - } - triggerMouseEvent(dom, "mousemove", x, 100); - triggerMouseEvent(dom, "click", 260, 100); - await waitFor(PubSubEvents.CLICK); - - const events = stopWatching().coreEvents; - assert.equal(events.length, 3); - assert.equal(events[0].state.event, "mousemove"); - assert.equal(events[0].state.x, x); - assert.equal(events[1].state.event, "mousemove"); - assert.equal(events[1].state.x, x); - assert.equal(events[2].state.event, "click"); - done(); - })); - - function triggerMouseEvent(target: EventTarget, type: string, x: number, y: number): void { - let mouseEvent; - if (typeof MouseEvent !== "function") { - mouseEvent = document.createEvent("MouseEvents"); - mouseEvent.initMouseEvent( - type, - true, // bubbles = true - false, // cancellable = false - window, - 0, - x, // screenX - y, // screenY - x, // clientX - y, // clientY - false, // ctrlKey - false, // altKey - false, // shftKey - false, // metaKey - 0, // button - null // relatedTarget - ); - } else { - mouseEvent = new MouseEvent(type, { - clientX: x, - clientY: y, - view: window, - bubbles: true, - cancelable: true - }); - } - target.dispatchEvent(mouseEvent); - } -}); diff --git a/package.json b/package.json index d1bfadab..f431eeeb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "0.3.0", + "version": "0.4.0", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", @@ -67,10 +67,6 @@ "build:cjs": "webpack --config webpack/configs/prod.ts", "build:cjs:dev": "webpack --config webpack/configs/dev.ts", "build:clean": "del-cli build/*", - "test": "karma start karma/configs/headless.js", - "test:chrome": "karma start karma/configs/chrome.js", - "test:firefox": "karma start karma/configs/firefox.js", - "test:ie": "karma start karma/configs/ie.js", "tslint": "tslint --project ./", "tslint:fix": "tslint --fix --project ./ --force" }, diff --git a/src/clarity.ts b/src/clarity.ts index fd06cd24..8fb3a837 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,21 +1,3 @@ -import { IConfig } from "@clarity-types/config"; -import { State } from "@clarity-types/core"; -import { config } from "./config"; -import { activate, onTrigger, state, teardown } from "./core"; -import { mapProperties } from "./utils"; -export { version } from "./core"; - -export function start(customConfig?: IConfig): void { - if (state !== State.Activated) { - mapProperties(customConfig, null, true, config); - activate(); - } -} - -export function stop(): void { - teardown(); -} - -export function trigger(key: string): void { - onTrigger(key); +export function start(): void { + console.log("clarity"); } diff --git a/src/compress.ts b/src/compress.ts deleted file mode 100644 index dbc3a562..00000000 --- a/src/compress.ts +++ /dev/null @@ -1,238 +0,0 @@ -/* Credit: http://pieroxy.net/blog/pages/lz-string/index.html */ - -// Excluding 3rd party code from tslint -/* tslint:disable */ - -// This library supports various encoding options (UTF16, standard UTF16, Base64, UINT8) -// And, different encoding choices impact the final compress bytes count -// By default, UTF16 which takes 16 bytes per character only work in WebKit browsers -// There's also an option for standard UTF16, which uses 15 bytes per character and -// that's efficient however payload encoded on iPhone can't be decoded using node.js -// For this reason, we are using Base64 at the moment. -export default function(uncompressed: string) { - var bitsPerChar = 6; - var keyStrBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; - // @ts-ignore: Implicity 'any' type - var getCharFromInt = function(a) { return keyStrBase64.charAt(a); } - if (uncompressed == null) return ""; - var i, value, - context_dictionary = {}, - context_dictionaryToCreate = {}, - context_c = "", - context_wc = "", - context_w = "", - context_enlargeIn = 2, // Compensate for the first entry which should not count - context_dictSize = 3, - context_numBits = 2, - context_data = [], - context_data_val = 0, - context_data_position = 0, - ii; - - for (ii = 0; ii < uncompressed.length; ii += 1) { - context_c = uncompressed.charAt(ii); - if (!Object.prototype.hasOwnProperty.call(context_dictionary, context_c)) { - context_dictionary[context_c] = context_dictSize++; - context_dictionaryToCreate[context_c] = true; - } - - context_wc = context_w + context_c; - if (Object.prototype.hasOwnProperty.call(context_dictionary, context_wc)) { - context_w = context_wc; - } else { - if (Object.prototype.hasOwnProperty.call(context_dictionaryToCreate, context_w)) { - if (context_w.charCodeAt(0) < 256) { - for (i = 0; i < context_numBits; i++) { - context_data_val = (context_data_val << 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - } - value = context_w.charCodeAt(0); - for (i = 0; i < 8; i++) { - context_data_val = (context_data_val << 1) | (value & 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = value >> 1; - } - } else { - value = 1; - for (i = 0; i < context_numBits; i++) { - context_data_val = (context_data_val << 1) | value; - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = 0; - } - value = context_w.charCodeAt(0); - for (i = 0; i < 16; i++) { - context_data_val = (context_data_val << 1) | (value & 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = value >> 1; - } - } - context_enlargeIn--; - if (context_enlargeIn == 0) { - context_enlargeIn = Math.pow(2, context_numBits); - context_numBits++; - } - delete context_dictionaryToCreate[context_w]; - } else { - value = context_dictionary[context_w]; - for (i = 0; i < context_numBits; i++) { - context_data_val = (context_data_val << 1) | (value & 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = value >> 1; - } - - - } - context_enlargeIn--; - if (context_enlargeIn == 0) { - context_enlargeIn = Math.pow(2, context_numBits); - context_numBits++; - } - // Add wc to the dictionary. - context_dictionary[context_wc] = context_dictSize++; - context_w = String(context_c); - } - } - - // Output the code for w. - if (context_w !== "") { - if (Object.prototype.hasOwnProperty.call(context_dictionaryToCreate, context_w)) { - if (context_w.charCodeAt(0) < 256) { - for (i = 0; i < context_numBits; i++) { - context_data_val = (context_data_val << 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - } - value = context_w.charCodeAt(0); - for (i = 0; i < 8; i++) { - context_data_val = (context_data_val << 1) | (value & 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = value >> 1; - } - } else { - value = 1; - for (i = 0; i < context_numBits; i++) { - context_data_val = (context_data_val << 1) | value; - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = 0; - } - value = context_w.charCodeAt(0); - for (i = 0; i < 16; i++) { - context_data_val = (context_data_val << 1) | (value & 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = value >> 1; - } - } - context_enlargeIn--; - if (context_enlargeIn == 0) { - context_enlargeIn = Math.pow(2, context_numBits); - context_numBits++; - } - delete context_dictionaryToCreate[context_w]; - } else { - value = context_dictionary[context_w]; - for (i = 0; i < context_numBits; i++) { - context_data_val = (context_data_val << 1) | (value & 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = value >> 1; - } - - - } - context_enlargeIn--; - if (context_enlargeIn == 0) { - context_enlargeIn = Math.pow(2, context_numBits); - context_numBits++; - } - } - - // Mark the end of the stream - value = 2; - for (i = 0; i < context_numBits; i++) { - context_data_val = (context_data_val << 1) | (value & 1); - if (context_data_position == bitsPerChar - 1) { - context_data_position = 0; - context_data.push(getCharFromInt(context_data_val)); - context_data_val = 0; - } else { - context_data_position++; - } - value = value >> 1; - } - - // Flush the last char - while (true) { - context_data_val = (context_data_val << 1); - if (context_data_position == bitsPerChar - 1) { - context_data.push(getCharFromInt(context_data_val)); - break; - } - else context_data_position++; - } - var res = context_data.join(''); - switch (res.length % 4) { // To produce valid Base64 - default: // When could this happen ? - case 0: return res; - case 1: return res + "==="; - case 2: return res + "=="; - case 3: return res + "="; - } -} diff --git a/src/compressionworker.ts b/src/compressionworker.ts deleted file mode 100644 index cbba5602..00000000 --- a/src/compressionworker.ts +++ /dev/null @@ -1,110 +0,0 @@ -import Compress from "./compress"; - -import { IAddEventMessage, ICompressedBatchMessage, ITimestampedWorkerMessage, WorkerMessageType } from "@clarity-types/compressionworker"; -import { IEnvelope, IEventArray, IPayload } from "@clarity-types/core"; -import { config as Config } from "./config"; - -export function createCompressionWorker( - envelope: IEnvelope, - onMessage?: (e: MessageEvent) => void, - onError?: (e: ErrorEvent) => void -): Worker { - let worker = null; - if (Worker) { - let workerUrl = createWorkerUrl(envelope); - worker = new Worker(workerUrl); - worker.onmessage = onMessage || null; - worker.onerror = onError || null; - } - return worker; -} - -function workerContext(): void { - let workerGlobalScope = self as any; - let compress = workerGlobalScope.compress; - let config = workerGlobalScope.config; - let envelope: IEnvelope = workerGlobalScope.envelope; - let nextBatchEvents: IEventArray[] = []; - let nextBatchBytes = 0; - let sequence = 0; - - // Edge case: Flag to skip uploading batches consisting of a single XhrError instrumentation event - // This helps us avoid the infinite loop in the case when all requests fail (e.g. dropped internet connection) - // Infinite loop comes from sending instrumentation about failing to deliver previous delivery failure instrumentation. - let nextBatchIsSingleXhrErrorEvent: boolean = false; - - self.onmessage = (evt: MessageEvent): void => { - let message = evt.data; - switch (message.type) { - case WorkerMessageType.AddEvent: - let addEventMsg = message as IAddEventMessage; - addEvent(addEventMsg.event, addEventMsg.time, addEventMsg.isXhrErrorEvent); - break; - case WorkerMessageType.ForceCompression: - let forceCompressionMsg = message as ITimestampedWorkerMessage; - postNextBatchToCore(forceCompressionMsg.time); - break; - default: - break; - } - }; - - function addEvent(event: IEventArray, time: number, isXhrErrorEvent?: boolean): void { - let eventStr = JSON.stringify(event); - - // If appending new event to next batch would exceed batch limit, then post next batch first - if (nextBatchBytes > 0 && nextBatchBytes + eventStr.length > config.batchLimit) { - postNextBatchToCore(time); - } - - // // Append new event to the next batch - nextBatchIsSingleXhrErrorEvent = (nextBatchEvents.length === 0 && isXhrErrorEvent); - nextBatchEvents.push(event); - nextBatchBytes += eventStr.length; - - // Even if we just posted next batch, it is possible that a single new event exceeds batch limit by itself, so we need to check again - if (nextBatchBytes >= config.batchLimit) { - postNextBatchToCore(time); - } - } - - function postNextBatchToCore(time: number): void { - if (nextBatchBytes > 0 && !nextBatchIsSingleXhrErrorEvent) { - envelope.sequenceNumber = sequence++; - envelope.time = time; - let raw: IPayload = { envelope, events: nextBatchEvents }; - let rawStr = JSON.stringify(raw); - let compressed = compress(rawStr); - nextBatchEvents = []; - nextBatchBytes = 0; - postToCore(compressed, raw); - } - } - - function postToCore(compressed: string, raw: IPayload): void { - let message: ICompressedBatchMessage = { - type: WorkerMessageType.CompressedBatch, - compressedData: compressed, - rawData: raw - }; - - // Post message to the main thread - workerGlobalScope.postMessage(message); - } -} - -// Workers are initialized with a URL, pointing to the code which is going to be executed within worker's scope. -// URL can point to file, however we don't want to load a file with worker's code separately, so we create a Blob -// with a string containing worker's code. To build such string, we stitch together string representations of -// all functions and objects that are going to be required within the worker's scope. -// Once Blob is created, we create a URL pointing to it, which can be passed to worker's constructor. -function createWorkerUrl(envelope: IEnvelope): string { - let workerContextStr = workerContext.toString(); - let workerStr = workerContextStr.substring(workerContextStr.indexOf("{") + 1, workerContextStr.lastIndexOf("}")); - let code = `self.compress=${Compress.toString()};` - + `self.config=${JSON.stringify(Config)};` - + `self.envelope=${JSON.stringify(envelope)};` - + workerStr; - let blob = new Blob([code], {type: "application/javascript"}); - return URL.createObjectURL(blob); -} diff --git a/src/config.ts b/src/config.ts deleted file mode 100644 index 7cd38f65..00000000 --- a/src/config.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { IConfig } from "@clarity-types/config"; -import { mapProperties } from "./utils"; - -// Default configuration -export let config: IConfig = { - plugins: ["viewport", "layout", "pointer", "performance", "errors"], - uploadUrl: "", - urlBlacklist: [], - delay: 500, - batchLimit: 100 * 1024, // 100 kilobytes - totalLimit: 20 * 1024 * 1024, // 20 megabytes - reUploadLimit: 1, - disableCookie: false, - sensitiveAttributes: ["value"], - instrument: false, - cssRules: false, - uploadHandler: null, - uploadHeaders: { - "Content-Type": "application/json" - }, - customInstrumentation: null, - debug: false, - validateConsistency: false, - backgroundMode: false -}; - -export function resetConfig(): void { - mapProperties(defaultConfig, null, false, config); -} - -let defaultConfig: IConfig = {}; -mapProperties(config, null, false, defaultConfig); diff --git a/src/converters/convert.ts b/src/converters/convert.ts deleted file mode 100644 index 9a311429..00000000 --- a/src/converters/convert.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { default as encode } from "./toarray"; -export { default as decode } from "./fromarray"; -export { SchemaManager } from "./schema"; diff --git a/src/converters/fromarray.ts b/src/converters/fromarray.ts deleted file mode 100644 index 56c00044..00000000 --- a/src/converters/fromarray.ts +++ /dev/null @@ -1,67 +0,0 @@ -import defaultSchemas from "./schema"; - -import { IEvent, IEventArray, ObjectType } from "@clarity-types/core"; -import { SchemaManager } from "./schema"; - -export default function(eventArray: IEventArray, schemas?: SchemaManager): IEvent { - let id, type, time, stateArray, schema; - [id, type, time, stateArray, schema] = eventArray; - - if (!schemas) { - schemas = defaultSchemas; - } - - if (typeof schema === "number") { - schema = schemas.getSchema(schema); - } else { - schemas.addSchema(schema); - } - - let state = dataFromArray(stateArray, schema as any[]); - let event: IEvent = { id, type, time, state }; - return event; -} - -// This function reconstructs the original object from array of data and a schema that describes it. Check schema.md for details: -// https://github.com/Microsoft/clarity-js/blob/master/converters/schema.md -function dataFromArray(dataArray: any[], schema: any[]): any { - // Schema is of type "string" or null when data that matches it is not an Array or an Object, - // so no additional reconstruction on the data is required. - if (typeof schema === "string" || schema === null) { - return dataArray; - } - - let data = null; - let dataType = null; - let subschemas = null; - if (schema.length === 2) { - // Schema is [ObjectType, [Array values' or Object properties' schemas (based on the object type) ]] - dataType = schema[0]; - subschemas = schema[1]; - } else if (schema.length === 3) { - // Schema is [Property name in parent object, ObjectType, [Array values' or Object properties' schemas (based on the object type) ]] - dataType = schema[1]; - subschemas = schema[2]; - } - - if (dataType === ObjectType.Object) { - data = {}; - for (let i = 0; i < subschemas.length; i++) { - let currentSubschema = subschemas[i]; - let currentProperty = null; - if (typeof currentSubschema === "string") { - currentProperty = currentSubschema; - } else { - currentProperty = currentSubschema[0]; - } - data[currentProperty] = dataFromArray(dataArray[i], currentSubschema); - } - } else if (dataType === ObjectType.Array) { - data = []; - for (let i = 0; i < subschemas.length; i++) { - let nextSubschema = subschemas[i]; - data.push(dataFromArray(dataArray[i], nextSubschema)); - } - } - return data; -} diff --git a/src/converters/schema.md b/src/converters/schema.md deleted file mode 100644 index 47800755..00000000 --- a/src/converters/schema.md +++ /dev/null @@ -1,66 +0,0 @@ -Schema represents the structure of an arbitrary JavaScript variable. For Objects, the goal is to list its property names without property values, as an array. - -The gist of the idea can be illustrated with a simple example: - -```javascript -let car = { - make: "Tesla", - year: 2018 -}; -schema(car) = ["make", "year"] -``` - -However, schemas needs to be able to handle more complex objects, which can have properties of arbitrary types, which might require their own schemas. When we think about these slightly harder cases, there are two issues that come up that need to be handled as special cases. - -### Handling nested Objects - -```javascript -let car = { - make: "Tesla", - year: 2018, - features: { - color: "red", - } -} -``` - -Now, to construct a schema for 'car', it's not enough to just say ["make, "year", "features"], because features has a schema of its own, which needs to be respected as well. To take that into account, we need to add an extra dimension to the features representation, which would allow us to pass two pieces of inofrmation for the features object - property name, under which it's stored in the car object and features's own schema. It can look something like this: - -featuresSchema = ["features", ["color"]] -schema(car) = [ "make", "year", featuresSchema ] - -### Differentiating Object's schema arrays from actual arrays of data - -Last problem that we need to handle is disambiguation between the arrays describing Objects schemas and arrays describing Array schemas. We can do it the same way we handle nested objects - by adding an extra dimension indicating what is the object type that is being described by this schema - Object or an Array. - -Here is a more complicated example containing three schema types (primitive, array, object) nested within another object. - -```javascript -let car = { - make: "Tesla", - year: 2018, - features: { - color: "red", - }, - passenges: [ - "John", - "Lena" - ] -} -``` - -featuresSchema = ["features", ObjectType.Object, ["color"]] -passengersSchema = ["passengers", ObjectType.Array, [null, null]]; -schema(car) = [ ObjectType.Object, [ "make", "year", featuresSchema, passengerSchema ] ] - -More generally speaking, schema is determined by (1) the type of variable for which schema is created, and (2) whether this variable is contained within some other object as a value for a property with a name. Table below contains all permutations of those two factors: - -[comment]: # (Markdown for table below is auto-generated and renders correctly) -[comment]: # (Generator URL: https://www.tablesgenerator.com/markdown_tables) - -| Is a named property value | Primitive | Array | Object | -|---------------------------|---------------|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------| -| FALSE | null | [ObjectType.Array, [Array of array values' recursive Schemas]] | [ObjectType.Object, [Array of array values' recursive Schemas]] | -| TRUE | Property name | [Property name, ObjectType.Array, [Array of object values' recursive Schemas]] | [Property name, ObjectType.Array, [Array of object values' recursive Schemas]] | - -[comment]: # (End of table) \ No newline at end of file diff --git a/src/converters/schema.ts b/src/converters/schema.ts deleted file mode 100644 index 64ef5f9b..00000000 --- a/src/converters/schema.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { ClarityDataSchema, ObjectType } from "@clarity-types/core"; - -// Details about schema generation are in schema.md: -// https://github.com/Microsoft/clarity-js/blob/master/converters/schema.md - -export class SchemaManager { - - private schemas: ClarityDataSchema[]; - private schemaToIdMap: { [key: string]: number }; - - constructor() { - this.reset(); - } - - public reset(): void { - this.schemas = []; - this.schemaToIdMap = {}; - } - - public addSchema(schema: ClarityDataSchema): boolean { - let schemaStr = JSON.stringify(schema); - let isNew = !(schemaStr in this.schemaToIdMap); - if (isNew) { - this.schemaToIdMap[schemaStr] = this.schemas.length; - this.schemas.push(schema); - } - return isNew; - } - - public createSchema(data: any, name?: string): ClarityDataSchema { - let schema: ClarityDataSchema = name || null; - if (data !== null) { - if (typeof data === "object" && (data.constructor === Object || data.constructor === Array)) { - // Objects with properties and Arrays require nested schema generation - schema = this.createNestedSchema(data, name); - } else { - // Primitives and simple objects can return schema immediately - schema = typeof name === "string" ? name : null; - } - } - return schema; - } - - public getSchema(schemaId: number): ClarityDataSchema { - return this.schemas[schemaId]; - } - - public getSchemaId(schema: ClarityDataSchema): number | undefined { - return this.schemaToIdMap[JSON.stringify(schema)]; - } - - private createNestedSchema(data: object | any[], name?: string): ClarityDataSchema { - let nestedObjectType = null; - let dataSchema = null; - if (data.constructor === Object) { - nestedObjectType = ObjectType.Object; - dataSchema = this.createObjectSchema(data); - } else if (data.constructor === Array) { - nestedObjectType = ObjectType.Array; - dataSchema = this.createArraySchema(data as any[]); - } - let schema = [nestedObjectType, dataSchema]; - if (typeof name === "string") { - schema.unshift(name); - } - return schema; - } - - private createObjectSchema(data: object): ClarityDataSchema { - let schema = []; - let keys = Object.keys(data); - for (let i = 0; i < keys.length; i++) { - let key = keys[i]; - let val = data[key]; - let subschema = this.createSchema(val, key); - schema.push(subschema); - } - return schema; - } - - private createArraySchema(data: any[]): ClarityDataSchema { - let schema = []; - for (let i = 0; i < data.length; i++) { - let subschema = this.createSchema(data[i]); - schema.push(subschema); - } - return schema; - } -} - -export function resetSchemas(): void { - schemaManager.reset(); -} - -let schemaManager: SchemaManager = new SchemaManager(); -export default schemaManager; diff --git a/src/converters/toarray.ts b/src/converters/toarray.ts deleted file mode 100644 index f694516b..00000000 --- a/src/converters/toarray.ts +++ /dev/null @@ -1,46 +0,0 @@ -import schemas from "./schema"; - -import { IEvent, IEventArray } from "@clarity-types/core"; - -// We serialize send a lot of JSONs with identical structures and putting all property names on the -// wire every time can be very redundant. We can save bytes by splitting JSON into a separate nested array of -// its property values and a separate nested array of its property names (schema). This way, each unique schema -// has to only be sent for the first event of that type and the following events can then re-use this schema -// on the server for reconstructing the full JSON. -export default function(event: IEvent): IEventArray { - let schema = schemas.createSchema(event.state); - let newSchema = schemas.addSchema(schema); - let stateArray = dataToArray(event.state); - let schemaPayload = newSchema ? schema : schemas.getSchemaId(schema); - let array = [event.id, event.type, event.time, stateArray, schemaPayload] as IEventArray; - return array; -} - -// Arbitrary JavaScript object can be represented as a value or an array of values without any property names -// Crunching objects to arrays is done the following way: -// 1. Array: Keep object an array, but recursively crunch its contents -// 2. Object with properties: Create an array of recursively crunched property values in the order returned by Object.keys -// Example: let car = { make: "tesla", model: "3"}. Crunched array would be ["tesla", "3"] -// 3. Other: Just send the value as is -function dataToArray(data: any): any[] { - if (data === null || typeof data !== "object") { - return data; - } - - let dataArray = []; - if (data.constructor === Object) { - let keys = Object.keys(data); - for (let i = 0; i < keys.length; i++) { - let key = keys[i]; - let val = data[key]; - dataArray.push(dataToArray(val)); - } - } else if (data.constructor === Array) { - for (let i = 0; i < data.length; i++) { - let arrayValue = dataToArray(data[i]); - dataArray.push(arrayValue); - } - } - - return dataArray; -} diff --git a/src/core.ts b/src/core.ts index 7526b202..732be94c 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,385 +1 @@ -import compress from "./compress"; -import EventToArray from "./converters/toarray"; -import getPlugin from "./plugins"; - -import { IAddEventMessage, ICompressedBatchMessage, ITimestampedWorkerMessage, WorkerMessageType } from "@clarity-types/compressionworker"; -import { - IBindingContainer, IClarityFields, IEnvelope, IEvent, IEventArray, IEventBindingPair, IEventData, IPayload, IPlugin, State -} from "@clarity-types/core"; -import { - IClarityActivateErrorState, IClarityDuplicatedEventState, IInstrumentationEventState, - IMissingFeatureEventState, Instrumentation, ITriggerState -} from "@clarity-types/instrumentation"; -import { createCompressionWorker } from "./compressionworker"; -import { config, resetConfig } from "./config"; -import { resetSchemas } from "./converters/schema"; -import { enqueuePayload, flushPayloadQueue, resetUploads, upload } from "./upload"; -import { getCookie, getEventId, guid, isNumber, setCookie } from "./utils"; - -export const version = "0.3.0"; -export const ClarityAttribute = "clarity-iid"; -export const InstrumentationEventName = "Instrumentation"; -const Cookie = "ClarityID"; - -let startTime: number; -let cid: string; -let impressionId: string; -let sequence: number; -let envelope: IEnvelope; -let activePlugins: IPlugin[]; -let bindings: IBindingContainer; - -// Counters -let eventCount: number; - -// Storage for events that were posted to compression worker, but have not returned to core as compressed batches yet. -// When page is unloaded, keeping such event copies in core allows us to terminate compression worker safely and then -// compress and upload remaining events synchronously from the main thread. -let pendingEvents: { [key: number]: IEventArray }; - -let backgroundMode: boolean; -let compressionWorker: Worker; -let timeout: number; - -export let state: State = State.Loaded; - -export function activate(): void { - state = State.Activating; - - // First, try to initalize core variables to allow Clarity perform minimal logging and safe teardowns. - // If this step fails, attempt a potentially unsafe logging and teardown. - try { - init(); - } catch (e) { - onActivateErrorUnsafe(e); - return; - } - - // Next, prepare for activation and activate available plugins. - // If anything goes wrong at this stage, we should be able to perform a safe teardown. - try { - let readyToActivatePlugins = prepare(); - if (readyToActivatePlugins) { - activatePlugins(); - } else { - teardown(); - return; - } - } catch (e) { - onActivateError(e); - return; - } - - state = State.Activated; -} - -export function teardown(): void { - if (state === State.Activating || state === State.Activated) { - state = State.Unloading; - for (let plugin of activePlugins) { - plugin.teardown(); - } - - // Walk through existing list of bindings and remove them all - for (let evt in bindings) { - if (bindings.hasOwnProperty(evt)) { - let eventBindings = bindings[evt] as IEventBindingPair[]; - for (let i = 0; i < eventBindings.length; i++) { - (eventBindings[i].target).removeEventListener(evt, eventBindings[i].listener); - } - } - } - - if (compressionWorker) { - // Immediately terminate the worker and kill its thread. - // Any possible pending incoming messages from the worker will be ignored in the 'Unloaded' state. - // Copies of all the events that were sent to the worker, but have not been returned as a compressed batch yet, - // are stored in the 'pendingEvents' object, so we will compress and upload them synchronously in this thread. - compressionWorker.terminate(); - } - state = State.Unloaded; - - // Instrument teardown and upload residual events - instrument({ type: Instrumentation.Teardown }); - uploadPendingEvents(); - resetConfig(); - - delete document[ClarityAttribute]; - } -} - -export function bind(target: EventTarget, event: string, listener: EventListener): void { - let eventBindings = bindings[event] || []; - target.addEventListener(event, listener, false); - eventBindings.push({ - target, - listener - }); - bindings[event] = eventBindings; -} - -export function addEvent(event: IEventData, scheduleUpload: boolean = true): void { - let evtJson: IEvent = { - id: eventCount++, - time: isNumber(event.time) ? event.time : getTimestamp(), - type: event.type, - state: event.state - }; - let evt = EventToArray(evtJson); - let addEventMessage: IAddEventMessage = { - type: WorkerMessageType.AddEvent, - event: evt, - time: getTimestamp(), - isXhrErrorEvent: event.type === InstrumentationEventName && event.state.type === Instrumentation.XhrError - }; - if (compressionWorker) { - compressionWorker.postMessage(addEventMessage); - } - pendingEvents[evtJson.id] = evt; - if (scheduleUpload) { - clearTimeout(timeout); - timeout = window.setTimeout(forceCompression, config.delay); - } -} - -export function addMultipleEvents(events: IEventData[]): void { - if (events.length > 0) { - // Don't schedule upload until we add the last event - for (let i = 0; i < events.length - 1; i++) { - addEvent(events[i], false); - } - let lastEvent = events[events.length - 1]; - addEvent(lastEvent, true); - } -} - -export function onTrigger(key: string): void { - if (state === State.Activated) { - let triggerState: ITriggerState = { - type: Instrumentation.Trigger, - key - }; - instrument(triggerState); - backgroundMode = false; - flushPayloadQueue(); - } -} - -export function forceCompression(): void { - if (compressionWorker) { - let forceCompressionMessage: ITimestampedWorkerMessage = { - type: WorkerMessageType.ForceCompression, - time: getTimestamp() - }; - compressionWorker.postMessage(forceCompressionMessage); - } -} - -export function getTimestamp(unix?: boolean, raw?: boolean): number { - let time = unix ? getUnixTimestamp() : getPageContextBasedTimestamp(); - return (raw ? time : Math.round(time)); -} - -export function instrument(eventState: IInstrumentationEventState): void { - if (config.instrument) { - addEvent({type: InstrumentationEventName, state: eventState}); - } -} - -export function onWorkerMessage(evt: MessageEvent): void { - if (state !== State.Unloaded) { - let message = evt.data; - switch (message.type) { - case WorkerMessageType.CompressedBatch: - let uploadMsg = message as ICompressedBatchMessage; - if (backgroundMode) { - enqueuePayload(uploadMsg.compressedData, uploadMsg.rawData); - } else { - upload(uploadMsg.compressedData, uploadMsg.rawData); - } - sequence = uploadMsg.rawData.envelope.sequenceNumber + 1; - - // Clear records for the compressed events returned by the worker - let events = uploadMsg.rawData.events; - for (let i = 0; i < events.length; i++) { - let evtId = getEventId(events[i]); - delete pendingEvents[evtId]; - } - break; - default: - break; - } - } -} - -function getUnixTimestamp(): number { - return (window.performance && performance.now && performance.timing) - ? performance.now() + performance.timing.navigationStart - : new Date().getTime(); -} - -// If performance.now function is not available, we do our best to approximate the time since page start -// by using the timestamp from when Clarity script got invoked as a starting point. -// In such case this number may not reflect the 'time since page start' accurately, -// especially if Clarity script is post-loaded or injected after page load. -function getPageContextBasedTimestamp(): number { - return (window.performance && performance.now) - ? performance.now() - : new Date().getTime() - startTime; -} - -function uploadPendingEvents(): void { - // We don't want to upload any data if Clarity is in background mode - if (backgroundMode) { - return; - } - let events: IEventArray[] = []; - let keys = Object.keys(pendingEvents); - for (let i = 0; i < keys.length; i++) { - let key = keys[i]; - let val = pendingEvents[key]; - events.push(val); - } - if (events.length > 0) { - envelope.sequenceNumber = sequence++; - envelope.time = getTimestamp(); - let raw: IPayload = { envelope, events }; - let compressed = compress(JSON.stringify(raw)); - upload(compressed, raw); - } -} - -function init(): void { - // Set ClarityID cookie, if it's not set already and is allowed by config - if (!config.disableCookie && !getCookie(Cookie)) { - // setting our ClarityId cookie for 2 years - setCookie(Cookie, guid(), 7 * 52 * 2); - } - - cid = config.disableCookie ? guid() : getCookie(Cookie); - impressionId = guid(); - - startTime = getUnixTimestamp(); - sequence = 0; - - envelope = { - clarityId: cid, - impressionId, - projectId: config.projectId || null, - url: window.location.href, - version - }; - - resetSchemas(); - resetUploads(); - - if (config.customInstrumentation) { - let fields: IClarityFields = { - impressionId, - clientId: cid, - projectId: config.projectId - }; - let customInst = config.customInstrumentation(fields); - envelope.extraInfo = {}; - for (let key in customInst) { - if (customInst.hasOwnProperty(key) && customInst[key]) { - envelope.extraInfo[key] = customInst[key]; - } - } - } - - activePlugins = []; - bindings = {}; - pendingEvents = []; - backgroundMode = config.backgroundMode; - - eventCount = 0; - - compressionWorker = createCompressionWorker(envelope, onWorkerMessage); -} - -function prepare(): boolean { - // If critical API is missing, don't activate Clarity - if (!checkFeatures()) { - return false; - } - - // Check that no other instance of Clarity is already running on the page - if (document[ClarityAttribute]) { - let eventState: IClarityDuplicatedEventState = { - type: Instrumentation.ClarityDuplicated, - currentImpressionId: document[ClarityAttribute] - }; - instrument(eventState); - return false; - } - - document[ClarityAttribute] = impressionId; - bind(window, "beforeunload", teardown); - bind(window, "unload", teardown); - return true; -} - -function activatePlugins(): void { - for (let plugin of config.plugins) { - let pluginClass = getPlugin(plugin); - if (pluginClass) { - let instance = new (pluginClass)(); - instance.reset(); - instance.activate(); - activePlugins.push(instance); - } - } -} - -function onActivateErrorUnsafe(e: Error): void { - try { - onActivateError(e); - } catch (e) { - // If there is an error at this stage, there is not much we can do any more, so just ignore and exit. - } -} - -function onActivateError(e: Error): void { - let clarityActivateError: IClarityActivateErrorState = { - type: Instrumentation.ClarityActivateError, - error: e.message - }; - instrument(clarityActivateError); - teardown(); -} - -function checkFeatures(): boolean { - let missingFeatures = []; - let expectedFeatures = [ - "document.implementation.createHTMLDocument", - "document.documentElement.classList", - "Function.prototype.bind", - "window.Worker" - ]; - - for (let feature of expectedFeatures) { - let parts = feature.split("."); - let api = window; - for (let part of parts) { - if (typeof api[part] === "undefined") { - missingFeatures.push(feature); - break; - } - api = api[part]; - } - } - - if (missingFeatures.length > 0) { - instrument({ - type: Instrumentation.MissingFeature, - missingFeatures - } as IMissingFeatureEventState); - return false; - } - - return true; -} - -// Initialize bindings early, so that registering and wiring up can be done properly -bindings = {}; +let exist = true; diff --git a/src/index.ts b/src/index.ts index 4e4d6b87..d84e780f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,3 @@ import * as ClarityJs from "./clarity"; -import * as PayloadEncoder from "./converters/convert"; -export { PayloadEncoder, ClarityJs }; +export { ClarityJs }; diff --git a/src/plugins.ts b/src/plugins.ts deleted file mode 100644 index d8e6ee85..00000000 --- a/src/plugins.ts +++ /dev/null @@ -1,19 +0,0 @@ -import errors from "./plugins/errors"; -import layout from "./plugins/layout/layout"; -import performance from "./plugins/performance"; -import pointer from "./plugins/pointer/pointer"; -import viewport from "./plugins/viewport"; - -type ClarityPlugin = typeof errors | typeof layout | typeof performance | typeof pointer | typeof viewport; - -const classes: { [key: string]: ClarityPlugin } = { - layout, - viewport, - pointer, - performance, - errors -}; - -export default function getPlugin(name: string): ClarityPlugin { - return classes[name]; -} diff --git a/src/plugins/errors.ts b/src/plugins/errors.ts deleted file mode 100644 index 98546f55..00000000 --- a/src/plugins/errors.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { IPlugin } from "@clarity-types/core"; -import { IJsErrorEventState, Instrumentation } from "@clarity-types/instrumentation"; -import { bind, instrument } from "@src/core"; - -export default class ErrorMonitor implements IPlugin { - - public activate(): void { - bind(window, "error", logError); - } - - public reset(): void { - return; - } - - public teardown(): void { - return; - } -} - -export function logError(errorToLog: Event): void { - // if errorToLog["error"] doesn't exist, occasionally we can get information directly from errorToLog - let error = errorToLog["error"] || errorToLog; - let source = errorToLog["filename"]; - let lineno = errorToLog["lineno"]; - let colno = errorToLog["colno"]; - let message = error.message; - let stack = error.stack; - - let jsErrorEventState: IJsErrorEventState = { - type: Instrumentation.JsError, - message, - stack, - lineno, - colno, - source - }; - instrument(jsErrorEventState); -} diff --git a/src/plugins/layout/layout.ts b/src/plugins/layout/layout.ts deleted file mode 100644 index 893ae101..00000000 --- a/src/plugins/layout/layout.ts +++ /dev/null @@ -1,344 +0,0 @@ -import { IEventData, IPlugin } from "@clarity-types/core"; -import { - IMutationPerformanceState, INodeStateGenPerformanceState, Instrumentation, IShadowDomInconsistentEventState -} from "@clarity-types/instrumentation"; -import { - Action, IElementLayoutState, ILayoutRoutineInfo, ILayoutState, IMutationRoutineInfo, InsertRuleHandler, - IShadowDomMutationSummary, IShadowDomNode, IStyleLayoutState, LayoutRoutine, NumberJson, Source -} from "@clarity-types/layout"; -import { config } from "@src/config"; -import { addEvent, addMultipleEvents, bind, getTimestamp, instrument } from "@src/core"; -import { debug, traverseNodeTree } from "@src/utils"; -import { ShadowDom } from "./shadowdom"; -import { getElementValue } from "./states/element"; -import { getNodeIndex, NodeIndex } from "./states/generic"; -import { resetStateProvider } from "./states/stateprovider"; -import { getCssRules } from "./states/style"; - -export default class Layout implements IPlugin { - private readonly eventName: string = "Layout"; - private readonly discoverPerfInstName: string = "Discover"; - private readonly mutationPerfInstName: string = "Mutation"; - private readonly cssTimeoutLength: number = 50; - private distanceThreshold: number = 5; - private shadowDom: ShadowDom; - private inconsistentShadowDomCount: number; - private observer: MutationObserver; - private insertRule: InsertRuleHandler; - private cssTimeout: number; - private cssElementQueue: Element[]; - private watchList: boolean[]; - private mutationSequence: number; - private lastConsistentDomJson: NumberJson; - private firstShadowDomInconsistentEvent: IShadowDomInconsistentEventState; - - public reset(): void { - this.shadowDom = new ShadowDom(); - this.inconsistentShadowDomCount = 0; - this.watchList = []; - this.cssElementQueue = []; - this.observer = window["MutationObserver"] ? new MutationObserver(this.mutation.bind(this)) : null; - this.insertRule = CSSStyleSheet.prototype.insertRule; - this.mutationSequence = 0; - this.lastConsistentDomJson = null; - this.firstShadowDomInconsistentEvent = null; - resetStateProvider(); - } - - public activate(): void { - this.discoverDom(); - if (this.observer) { - this.observer.observe(document, { - attributes: true, - childList: true, - characterData: true, - subtree: true - }); - } - if (this.insertRule) { - let that = this; - - // Some popular open source libraries, like styled-components, optimize performance - // by injecting CSS using insertRule API vs. appending text node. A side effect of - // using javascript API is that it doesn't trigger DOM mutation and therefore we - // need to override the insertRule API and listen for changes manually. - CSSStyleSheet.prototype.insertRule = function(rule: string, index?: number): number { - let value = that.insertRule.call(this, rule, index); - that.queueCss(this.ownerNode); - return value; - }; - } - } - - public teardown(): void { - if (this.observer) { - this.observer.disconnect(); - } - - if (this.cssTimeout) { - clearTimeout(this.cssTimeout); - } - - // Restore original insertRule definition - if (this.insertRule) { - CSSStyleSheet.prototype.insertRule = this.insertRule; - } - this.insertRule = null; - - // Clean up node indices on observed nodes - // If Clarity is re-activated within the same page later, - // old, uncleared indices would cause it to work incorrectly - let documentShadowNode = this.shadowDom.shadowDocument; - if (documentShadowNode.node) { - delete documentShadowNode.node[NodeIndex]; - } - let otherNodes = this.shadowDom.shadowDocument.querySelectorAll("*"); - for (let i = 0; i < otherNodes.length; i++) { - let node = (otherNodes[i] as IShadowDomNode).node; - if (node) { - delete node[NodeIndex]; - } - } - } - - private discoverDom(): void { - // All 'Discover' events together should be treated as an atomic 'Discover' operation and should have the same timestamp - const discoverStartTime = getTimestamp(); - let nodeCount = 0; - traverseNodeTree(document, (node: Node) => { - let shadowNode = this.discoverNode(node); - shadowNode.state.action = Action.Insert; - shadowNode.state.source = Source.Discover; - addEvent({ - type: this.eventName, - state: shadowNode.state, - time: discoverStartTime, - }); - nodeCount++; - }); - const discoverPerfState: INodeStateGenPerformanceState = { - type: Instrumentation.Performance, - procedure: this.discoverPerfInstName, - duration: getTimestamp() - discoverStartTime, - nodeCount - }; - instrument(discoverPerfState); - this.checkConsistency({ - action: LayoutRoutine.DiscoverDom - }); - } - - // Add node to the ShadowDom to store initial adjacent node info in a layout and obtain an index - private discoverNode(node: Node): IShadowDomNode { - let shadowNode = this.shadowDom.insertShadowNode(node, getNodeIndex(node.parentNode), getNodeIndex(node.nextSibling)); - shadowNode.computeInfo(); - shadowNode.computeState(); - this.watch(node, shadowNode.state); - return shadowNode; - } - - private watch(node: Node, nodeLayoutState: ILayoutState): void { - - // We only wish to watch elements once and then wait on the events to push changes - if (node.nodeType !== Node.ELEMENT_NODE || this.watchList[nodeLayoutState.index]) { - return; - } - - let element = node as Element; - let layoutState = nodeLayoutState as IElementLayoutState; - let scrollPossible = (layoutState.layout - && ("scrollX" in layoutState.layout - || "scrollY" in layoutState.layout)); - - if (scrollPossible) { - bind(element, "scroll", this.layoutHandler.bind(this, element, Source.Scroll)); - this.watchList[layoutState.index] = true; - } - - // Check if we need to monitor changes on input fields - if (element.tagName === "INPUT" || element.tagName === "SELECT") { - bind(element, "change", this.layoutHandler.bind(this, element, Source.Input)); - this.watchList[layoutState.index] = true; - } else if (element.tagName === "TEXTAREA") { - bind(element, "input", this.layoutHandler.bind(this, element, Source.Input)); - this.watchList[layoutState.index] = true; - } - } - - private queueCss(element: Element): void { - // Clear the timeout if it already exists - if (this.cssTimeout) { - clearTimeout(this.cssTimeout); - } - - // Queue element to be processed after the timeout triggers - if (this.cssElementQueue.indexOf(element) === -1) { - this.cssElementQueue.push(element); - } - - this.cssTimeout = window.setTimeout(this.cssDequeue.bind(this), this.cssTimeoutLength); - } - - private cssDequeue(): void { - for (let element of this.cssElementQueue) { - this.layoutHandler(element, Source.Css); - } - } - - private layoutHandler(element: Element, source: Source): void { - let index = getNodeIndex(element); - let shadowNode = this.shadowDom.getShadowNode(index); - if (shadowNode) { - let layoutState = shadowNode.state as IElementLayoutState; - switch (source) { - case Source.Scroll: - let scrollX = Math.round(element.scrollLeft); - let scrollY = Math.round(element.scrollTop); - let dx = layoutState.layout.scrollX - scrollX; - let dy = layoutState.layout.scrollY - scrollY; - if (dx * dx + dy * dy > this.distanceThreshold * this.distanceThreshold) { - layoutState.source = source; - layoutState.action = Action.Update; - layoutState.layout.scrollX = scrollX; - layoutState.layout.scrollY = scrollY; - addEvent({type: this.eventName, state: layoutState}); - } - break; - case Source.Input: - let input = element as HTMLInputElement; - layoutState.value = getElementValue(input, shadowNode.info); - layoutState.source = source; - layoutState.action = Action.Update; - addEvent({type: this.eventName, state: layoutState}); - break; - case Source.Css: - let styleState = shadowNode.state as IStyleLayoutState; - styleState.cssRules = getCssRules(element as HTMLStyleElement); - styleState.source = source; - styleState.action = Action.Update; - addEvent({type: this.eventName, state: styleState}); - break; - default: - break; - } - } - } - - private mutation(mutations: MutationRecord[]): void { - - const mutationStartTime = getTimestamp(); - let stateGenDuration = 0; - let summaryCounts: IMutationPerformanceState["summaryCounts"] = null; - - // Don't process mutations on top of the inconsistent state. - // ShadowDom mutation processing logic requires consistent state as a prerequisite. - // If we end up in the inconsistent state, that means that something went wrong already, - // so we can give up on the following mutations and should investigate the cause of the error. - // Continuing to process mutations can result in javascript errors and lead to even more inconsistencies. - if (this.allowMutation()) { - - // Perform mutations on the shadow DOM and make sure ShadowDom arrived to the consistent state - let summary = this.shadowDom.applyMutationBatch(mutations); - let actionInfo: IMutationRoutineInfo = { - action: LayoutRoutine.Mutation, - mutationSequence: this.mutationSequence, - batchSize: mutations.length - }; - summaryCounts = { - inserts: summary.newNodes.length, - moves: summary.movedNodes.length, - updates: summary.updatedNodes.length, - removes: summary.removedNodes.length - }; - this.checkConsistency(actionInfo); - if (this.allowMutation()) { - const stateGenStartTime = getTimestamp(); - this.processMutations(summary, mutationStartTime); - stateGenDuration = getTimestamp() - stateGenStartTime; - } - } - - const mutationPerfState: IMutationPerformanceState = { - type: Instrumentation.Performance, - procedure: this.mutationPerfInstName, - duration: getTimestamp() - mutationStartTime, - mutationCount: mutations.length, - mutationSequence: this.mutationSequence, - stateGenDuration, - summaryCounts - }; - instrument(mutationPerfState); - this.mutationSequence++; - } - - private allowMutation(): boolean { - return this.inconsistentShadowDomCount < 2 || !config.validateConsistency; - } - - private processMutations(summary: IShadowDomMutationSummary, time: number): void { - let inserts = summary.newNodes.map(this.processMutation.bind(this, Action.Insert)); - let moves = summary.movedNodes.map(this.processMutation.bind(this, Action.Move)); - let updates = summary.updatedNodes.map(this.processMutation.bind(this, Action.Update)); - let removes = summary.removedNodes.map(this.processMutation.bind(this, Action.Remove)); - let all: IEventData[] = [].concat(inserts, moves, updates, removes); - - // Since we receive batched mutations and reprocess them after, all mutation events from the same batch - // should be treated as a single atomic operation, so all of those events should have the same timestamp - for (let i = 0; i < all.length; i++) { - all[i].time = time; - } - addMultipleEvents(all); - } - - private processMutation(action: Action, shadowNode: IShadowDomNode): IEventData { - shadowNode.computeInfo(); - let state = shadowNode.computeState(); - state.action = action; - state.source = Source.Mutation; - state.mutationSequence = this.mutationSequence; - - // Watch new or updated nodes - if (action === Action.Insert || action === Action.Update) { - this.watch(shadowNode.node, state); - } else if (action === Action.Remove) { - // Removed nodes don't have an index any more, so computed state index will be null, - // however its original index can still be obtained from its matching shadow node id - state.index = parseInt(shadowNode.id, 10); - } - - return { type: this.eventName, state }; - } - - private checkConsistency(lastActionInfo: ILayoutRoutineInfo): void { - if (config.validateConsistency) { - let domJson = this.shadowDom.createIndexJson(document, (node: Node) => { - return getNodeIndex(node); - }); - let shadowDomConsistent = this.shadowDom.isConsistent(); - if (!shadowDomConsistent) { - this.inconsistentShadowDomCount++; - let shadowDomJson = this.shadowDom.createIndexJson(this.shadowDom.shadowDocument, (node: Node) => { - return parseInt((node as IShadowDomNode).id, 10); - }); - let evt: IShadowDomInconsistentEventState = { - type: Instrumentation.ShadowDomInconsistent, - dom: domJson, - shadowDom: shadowDomJson, - lastConsistentShadowDom: this.lastConsistentDomJson, - lastAction: lastActionInfo - }; - if (this.inconsistentShadowDomCount < 2) { - this.firstShadowDomInconsistentEvent = evt; - } else { - evt.firstEvent = this.firstShadowDomInconsistentEvent; - instrument(evt); - } - debug(`>>> ShadowDom doesn't match PageDOM after mutation batch #${this.mutationSequence}!`); - } else { - this.inconsistentShadowDomCount = 0; - this.firstShadowDomInconsistentEvent = null; - this.lastConsistentDomJson = domJson; - } - } - } -} diff --git a/src/plugins/layout/nodeinfo.ts b/src/plugins/layout/nodeinfo.ts deleted file mode 100644 index 41f79dde..00000000 --- a/src/plugins/layout/nodeinfo.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { INodeInfo } from "@clarity-types/layout"; -import { shouldIgnoreNode } from "./states/ignore"; -import { shouldCaptureCssRules } from "./states/style"; -import { isCssText } from "./states/text"; - -export const UnmaskAttribute = "data-clarity-unmask"; - -export function createNodeInfo(node: Node, parentInfo: INodeInfo): INodeInfo { - let ignore = shouldIgnoreNode(node, parentInfo); - let unmask = shouldUnmaskNode(node, parentInfo); - let isCss = isCssText(node); - let captureCssRules = shouldCaptureCssRules(node); - return { ignore, unmask, isCss, captureCssRules }; -} - -function shouldUnmaskNode(node: Node, parentInfo: INodeInfo): boolean { - const parentUnmasked = parentInfo ? parentInfo.unmask : false; - const hasUnmaskAttribute = ( - node && node.nodeType === Node.ELEMENT_NODE - ? (node as Element).getAttribute(UnmaskAttribute) === "true" - : false - ); - return hasUnmaskAttribute || parentUnmasked; -} diff --git a/src/plugins/layout/shadowdom.ts b/src/plugins/layout/shadowdom.ts deleted file mode 100644 index 2f5cedee..00000000 --- a/src/plugins/layout/shadowdom.ts +++ /dev/null @@ -1,412 +0,0 @@ -import { ILayoutState, INodeInfo, IShadowDomMutationSummary, IShadowDomNode, NumberJson } from "@clarity-types/layout"; -import { assert, isNumber, traverseNodeTree } from "@src/utils"; -import { createNodeInfo } from "./nodeinfo"; -import { getNodeIndex, NodeIndex } from "./states/generic"; -import { createLayoutState } from "./states/stateprovider"; - -// Class names to tag actions that happen to nodes in a single mutation batch -const FinalClassName = "cl-final"; -const NewNodeClassName = "cl-new"; -const MovedNodeClassName = "cl-moved"; -const UpdatedNodeClassName = "cl-updated"; - -export class ShadowDom { - public shadowDocument: IShadowDomNode; - - private doc: Document = document.implementation.createHTMLDocument("ShadowDom"); - private nextIndex: number = 0; - private removedNodes: HTMLDivElement = this.doc.createElement("div"); - private shadowDomRoot: HTMLDivElement = this.doc.createElement("div"); - private classifyNodes: boolean = false; - private nodeMap: { [key: number]: IShadowDomNode } = {}; - - constructor() { - this.doc.documentElement.appendChild(this.shadowDomRoot); - this.shadowDocument = document.createElement("div") as IShadowDomNode; - } - - public getShadowNode(index: number): IShadowDomNode { - let node = isNumber(index) ? this.nodeMap[index] : null; - return node; - } - - public insertShadowNode(node: Node, parentIndex: number, nextSiblingIndex: number): IShadowDomNode { - let isDocument = (node === document); - let index = this.assignNodeIndex(node); - let parent = (isDocument ? this.shadowDomRoot : this.getShadowNode(parentIndex)) as IShadowDomNode; - let nextSibling = this.getShadowNode(nextSiblingIndex); - let shadowNode = this.doc.createElement("div") as IShadowDomNode; - shadowNode.id = "" + index; - shadowNode.node = node; - - shadowNode.computeInfo = (): INodeInfo => { - const parentShadowNode = shadowNode.parentNode as IShadowDomNode; - shadowNode.info = createNodeInfo(node, parentShadowNode && parentShadowNode.info); - return shadowNode.info; - }; - shadowNode.computeState = (): ILayoutState => { - shadowNode.state = createLayoutState(node, shadowNode.info); - return shadowNode.state; - }; - this.nodeMap[index] = shadowNode; - - if (isDocument) { - this.shadowDocument = shadowNode; - } - - if (this.classifyNodes) { - this.setClass(shadowNode, NewNodeClassName); - } - - assert(!!parent, "insertShadowNode", "parent is missing"); - if (parent) { - if (nextSibling) { - parent.insertBefore(shadowNode, nextSibling); - } else { - parent.appendChild(shadowNode); - } - } - - return shadowNode; - } - - public moveShadowNode(index: number, newParentIndex: number, newNextSiblingIndex: number): IShadowDomNode { - let shadowNode = this.getShadowNode(index); - let parent = this.getShadowNode(newParentIndex); - let nextSibling = this.getShadowNode(newNextSiblingIndex); - - assert(!!parent, "moveShadowNode", "parent is missing"); - assert(!!shadowNode, "moveShadowNode", "shadowNode is missing"); - if (parent && shadowNode) { - if (this.classifyNodes) { - if (!this.hasClass(shadowNode, NewNodeClassName)) { - this.setClass(shadowNode, MovedNodeClassName); - } - } - - if (nextSibling) { - parent.insertBefore(shadowNode, nextSibling); - } else { - parent.appendChild(shadowNode); - } - } - return shadowNode; - } - - public updateShadowNode(index: number): void { - let shadowNode = this.getShadowNode(index); - if (shadowNode && this.classifyNodes) { - this.setClass(shadowNode, UpdatedNodeClassName); - } - } - - public removeShadowNode(index: number): void { - let shadowNode = this.getShadowNode(index); - if (shadowNode) { - this.setClass(shadowNode, MovedNodeClassName); - this.removedNodes.appendChild(shadowNode); - } - } - - // As we process a batch of mutations, various things can be happening to a single node - // In the end, however, for each affected node we will have one of the following outcomes: - // 1. A new node was added - // 2. Existing node was moved - // 3. Existing node was removed - // 4. Existing node was updated - // 5. Existing node was moved and updated - // Various actions on a node will lead to different finals states. Also a final state of a parent node - // can affect and/or override the state of its children (e.g. if node A was moved to node B, but B was removed, - // that means that A was removed as well). One of the simpler solutions to determining the final states of all - // affected nodes and its children is 'tagging' shadow nodes with class names based on the mutations as they happen, - // and then determining the final states of all affected nodes based on the combination of their classes (getting mutation summary). - // Note: When we process 'remove' action on a node, instead of marking it as removed and permanently removing it from ShadowDOM - // and erasing its index, we actually moved it to a separate 'removedNodes' container, which is not part of the representation - // of the real DOM, but is still a part of the document. This way, if this node is re-inserted to the page, we can just move - // add it back to ShadowDOM and just record the 'Move' event, letting it maintain its index. - public applyMutationBatch(mutations: MutationRecord[]): IShadowDomMutationSummary { - let nextIndexBeforeProcessing = this.nextIndex; - this.doc.documentElement.appendChild(this.removedNodes); - this.classifyNodes = true; - - let length = mutations.length; - for (let i = 0; i < length; i++) { - let mutation = mutations[i]; - let target = mutation.target; - switch (mutation.type) { - case "attributes": - case "characterData": - this.applyUpdate(target, mutation.attributeName, mutation.oldValue); - break; - case "childList": - - // Process inserts - // We use insertBefore to insert nodes into the shadowDom, so the right sibling needs to be inserted - // before the left sibling. For that reason we process elements from last to first (right to left) - let addedLength = mutation.addedNodes.length; - for (let j = addedLength - 1; j >= 0; j--) { - let previous = mutation.previousSibling; - let next = mutation.nextSibling; - if (j > 0) { - previous = mutation.addedNodes[j - 1]; - } - if (j < addedLength - 1) { - next = mutation.addedNodes[j + 1]; - } - this.applyInsert(mutation.addedNodes[j], target, previous, next, false); - } - - // Process removes - let removedLength = mutation.removedNodes.length; - for (let j = 0; j < removedLength; j++) { - this.applyRemove(mutation.removedNodes[j], target); - } - break; - default: - break; - } - } - - // Detach removed nodes - this.removedNodes.parentElement.removeChild(this.removedNodes); - - // Process the new state of the ShadowDom and extract the summary - let summary = this.getMutationSummary(); - - // Clear references to removed nodes and reset shadow dom state to get it ready for the next mutation batch - this.cleanUp(); - - // Re-assign indices for all new nodes that remained attached to DOM such that there are no gaps between them - this.reIndexNewNodes(summary.newNodes, nextIndexBeforeProcessing); - - return summary; - } - - public hasClass(shadowNode: IShadowDomNode, className: string): boolean { - return shadowNode ? shadowNode.classList.contains(className) : false; - } - - public setClass(shadowNode: IShadowDomNode, className: string): void { - if (shadowNode) { - shadowNode.classList.add(className); - } - } - - public removeClass(shadowNode: IShadowDomNode, className: string): void { - if (shadowNode) { - shadowNode.classList.remove(className); - } - } - - public removeAllClasses(shadowNode: IShadowDomNode): void { - if (shadowNode) { - shadowNode.removeAttribute("class"); - } - } - - // As a result of processing mutation batch, some shadow nodes that were affected by mutations have indicative class names. - // Steps to creating a mutation summary from these nodes are the following: - // 1. Record 'Insert' events on nodes with NewNodeClassName. We can ignore and remove other classes on such nodes, because it - // doesn't matter if this node was also moved around or updated - it's new, so we record its state from scratch anyways. - // 2. Record 'Move' events on remaining nodes with MovedNodeClassName class. - // 3. Record 'Update' events on remaining nodes with UpdatedNodeClassName class (there can be nodes that were moved AND updated). - // 4. Inspect nodes inside the 'removedNodes' container: - // - Ignore nodes that have NewNodeClassName class. They are new and were not in the ShadowDom to begin with - // - Record a 'Remove' event for nodes that have MovedNodeClassName. They were explicitly moved and ended up in the 'removed' container - // - Ignore remaining nodes. Since they weren't explicitly moved, they will be auto-removed through the subtree of their removed parent - public getMutationSummary(): IShadowDomMutationSummary { - let summary: IShadowDomMutationSummary = { - newNodes: [], - movedNodes: [], - updatedNodes: [], - removedNodes: [] - }; - - // Collect all new nodes in the top-down order - let newNodes = Array.prototype.slice.call(this.doc.getElementsByClassName(NewNodeClassName)); - for (let i = 0; i < newNodes.length; i++) { - let newNode = newNodes[i] as IShadowDomNode; - summary.newNodes.push(newNode); - this.removeAllClasses(newNode); - } - - let moved = Array.prototype.slice.call(this.doc.getElementsByClassName(MovedNodeClassName)); - for (let i = 0; i < moved.length; i++) { - let next = moved[i] as IShadowDomNode; - summary.movedNodes.push(next); - this.removeClass(next, MovedNodeClassName); - } - - let updated = Array.prototype.slice.call(this.doc.getElementsByClassName(UpdatedNodeClassName)); - for (let i = 0; i < updated.length; i++) { - let next = updated[i] as IShadowDomNode; - summary.updatedNodes.push(next); - this.removeAllClasses(next); - } - - traverseNodeTree(this.removedNodes, (removedNode: IShadowDomNode) => { - if (this.hasClass(removedNode, MovedNodeClassName) && !this.hasClass(removedNode, NewNodeClassName)) { - summary.removedNodes.push(removedNode); - } - }, false); - - return summary; - } - - public createIndexJson(rootNode: Node, getIndexFromNode: (node: Node) => number): NumberJson { - let indexJson: NumberJson = []; - this.writeIndexToJson(rootNode, indexJson, getIndexFromNode); - return indexJson; - } - - public isConsistent(): boolean { - return this.isConstentSubtree(document, this.shadowDocument); - } - - private cleanUp(): void { - - // For each removed dom node, remove its index and its shadow dom reference - traverseNodeTree(this.removedNodes, (removedNode: IShadowDomNode) => { - let index = getNodeIndex(removedNode.node); - delete removedNode.node[NodeIndex]; - delete this.nodeMap[index]; - }, false); - - // Reset the state of the shadow dom to be ready for next mutation batch processing - let finalNodes = Array.prototype.slice.call(this.doc.getElementsByClassName(FinalClassName)); - for (let i = 0; i < finalNodes.length; i++) { - this.removeAllClasses(finalNodes[i]); - } - this.removedNodes.innerHTML = ""; - this.classifyNodes = false; - } - - private writeIndexToJson(node: Node, json: NumberJson, getIndexFromNode: (node: Node) => number): void { - let index = getIndexFromNode(node); - let childJson: number[] = []; - let nextChild = node.firstChild; - json.push(index); - if (nextChild) { - json.push(childJson); - } - while (nextChild) { - this.writeIndexToJson(nextChild, childJson, getIndexFromNode); - nextChild = nextChild.nextSibling as IShadowDomNode; - } - } - - private isConsistentNode(node: Node, shadowNode: IShadowDomNode): boolean { - let index = getNodeIndex(node); - return (isNumber(index) && shadowNode.id === (index).toString() && shadowNode.node === node); - } - - private isConstentSubtree(node: Node, shadowNode: IShadowDomNode): boolean { - let isConsistent = this.isConsistentNode(node, shadowNode); - let nextChild: Node = node.firstChild; - let nextShadowChild: Node = shadowNode.firstChild; - while (isConsistent) { - if (nextChild && nextShadowChild) { - isConsistent = this.isConstentSubtree(nextChild, nextShadowChild as IShadowDomNode); - nextChild = nextChild.nextSibling; - nextShadowChild = nextShadowChild.nextSibling; - } else if (nextChild || nextShadowChild) { - isConsistent = false; - } else { - break; - } - } - return isConsistent; - } - - private applyInsert(addedNode: Node, parent: Node, previousSibling: Node, nextSibling: Node, force: boolean): void { - let addedNodeIndex = getNodeIndex(addedNode); - let parentIndex = getNodeIndex(parent); - let nextSiblingIndex = getNodeIndex(nextSibling); - let validMutation = this.shouldProcessChildListMutation(addedNode, parent) || force; - if (validMutation) { - // If inserted node has no index, then it's a new node and we should process insert - if (addedNodeIndex === null) { - let shadowNode = this.insertShadowNode(addedNode, parentIndex, nextSiblingIndex); - this.setClass(shadowNode, FinalClassName); - - // Process children - // We use insertBefore to insert nodes into the shadowDom, so the right sibling needs to be inserted - // before the left sibling. For that reason we process children from last to first (right to left) - let nextChild: Node = addedNode.lastChild; - while (nextChild) { - this.applyInsert(nextChild, addedNode, nextChild.previousSibling, nextChild.nextSibling, true); - nextChild = nextChild.previousSibling; - } - } else { - this.moveShadowNode(addedNodeIndex, parentIndex, getNodeIndex(nextSibling)); - } - } - } - - private applyRemove(removedNode: Node, parent: Node): void { - let removedNodeIndex = getNodeIndex(removedNode); - if (removedNodeIndex !== null) { - let validMutation = this.shouldProcessChildListMutation(removedNode, parent); - if (validMutation) { - this.removeShadowNode(removedNodeIndex); - } - } - } - - private applyUpdate(updatedNode: Node, attrName: string, oldValue: string): void { - let updatedNodeIndex = getNodeIndex(updatedNode); - if (updatedNodeIndex != null) { - this.updateShadowNode(updatedNodeIndex); - } - } - - // We want to determine whether we can skip this mutation without losing data. We can do so in 2 cases: - // 1. This is a mutation for a node that is marked as 'Final'. These are the new nodes for which we already know final position, - // because we have discovered them by traversing other inserted node's subtree in the real page DOM. For such final nodes, - // we already recorded the insert action to the appropriate position, so all other mutations can be ignored. - // 2. This is a mutation, which attempts to add or remove a node from the child list of the node, which is marked final. For such - // nodes, we have already processed their entire child list in the real page DOM and all children received an insert action. - // This means that any other mutations are either temporary (insert something that will end up being removed) or redundant, so - // we can skip them - private shouldProcessChildListMutation(child: Node, parent: Node): boolean { - let childNodeIndex = getNodeIndex(child); - let parentIndex = getNodeIndex(parent); - let parentShadowNode: IShadowDomNode = null; - if (childNodeIndex === null) { - parentShadowNode = this.getShadowNode(parentIndex); - } else { - let childShadowNode = this.getShadowNode(childNodeIndex); - parentShadowNode = childShadowNode && childShadowNode.parentNode as IShadowDomNode; - } - return parentShadowNode && !this.hasClass(parentShadowNode, FinalClassName); - } - - // When we apply a mutation batch, we assign a new index to every node that is added to the DOM as we parse mutations - // in their natural order. However, some of those nodes end up being again removed from DOM with some follow-up mutation - // thus 'stealing' an index with them. Since we don't instrument nodes that were added and removed within the same - // mutation batch, we have no records of these stolen indices. To maintain consistency on the backend, at the end of the mutation, - // we can re-assign indices for all nodes that remained attached to the DOM such that they all go in increasing order without gaps - private reIndexNewNodes(newNodes: IShadowDomNode[], nextIndex: number): void { - newNodes.map((shadowNode: IShadowDomNode) => { - let index = getNodeIndex(shadowNode.node); - delete this.nodeMap[index]; - }); - newNodes.map((shadowNode: IShadowDomNode) => { - shadowNode.node[NodeIndex] = nextIndex; - shadowNode.id = `${nextIndex}`; - this.nodeMap[nextIndex] = shadowNode; - nextIndex++; - }); - this.nextIndex = nextIndex; - } - - private assignNodeIndex(node: Node): number { - let index = getNodeIndex(node); - if (index === null) { - index = this.nextIndex; - this.nextIndex++; - } - node[NodeIndex] = index; - return index; - } -} diff --git a/src/plugins/layout/states/doctype.ts b/src/plugins/layout/states/doctype.ts deleted file mode 100644 index 75d4d9a2..00000000 --- a/src/plugins/layout/states/doctype.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { IDoctypeLayoutState } from "@clarity-types/layout"; -import { createGenericLayoutState, Tags } from "./generic"; - -export function createDoctypeLayoutState(doctypeNode: DocumentType): IDoctypeLayoutState { - let doctypeState = createGenericLayoutState(doctypeNode, Tags.Doc) as IDoctypeLayoutState; - doctypeState.attributes = { - name: doctypeNode.name, - publicId: doctypeNode.publicId, - systemId: doctypeNode.systemId - }; - return doctypeState; -} diff --git a/src/plugins/layout/states/element.ts b/src/plugins/layout/states/element.ts deleted file mode 100644 index f185eec4..00000000 --- a/src/plugins/layout/states/element.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { IAttributes, IElementLayoutState, ILayoutRectangle, ILayoutStyle, INodeInfo } from "@clarity-types/layout"; -import { config } from "@src/config"; -import { mask } from "@src/utils"; -import { createGenericLayoutState, Tags } from "./generic"; - -enum Styles { - Color = "color", - BackgroundColor = "backgroundColor", - BackgroundImage = "backgroundImage", - OverflowX = "overflowX", - OverflowY = "overflowY", - Visibility = "visibility" -} - -let defaultColor: string; -let attributeMaskList: string[]; - -const DefaultAttributeMaskList = ["value", "placeholder", "alt", "title"]; - -export function createElementLayoutState(element: Element, info: INodeInfo): IElementLayoutState { - let tagName = element.tagName; - let elementState = createGenericLayoutState(element, tagName) as IElementLayoutState; - if (tagName === Tags.Script || tagName === Tags.Meta) { - elementState.tag = Tags.Ignore; - return elementState; - } - - // Get attributes for the element - elementState.attributes = getAttributes(element, info); - - // Get layout bounding box for the element - elementState.layout = getLayout(element); - - // Get computed systems for the element with valid layout - elementState.style = elementState.layout ? getStyles(element) : null; - - // Check if scroll is possible - if (elementState.layout && elementState.style && (Styles.OverflowX in elementState.style || Styles.OverflowX in elementState.style)) { - elementState.layout.scrollX = Math.round(element.scrollLeft); - elementState.layout.scrollY = Math.round(element.scrollTop); - } - - // Certain elements might contain 'value' field, which means different things for different types of elements, - // but is ultimately important for each of them, so we must capture it as well. It is expected to be a string. - // https://www.w3schools.com/tags/att_value.asp (+ HTMLTextAreaElement) - if ("value" in element && typeof (element as any).value === "string") { - elementState.value = getElementValue(element, info); - } - - return elementState; -} - -export function resetElementStateProvider(): void { - attributeMaskList = DefaultAttributeMaskList.concat(config.sensitiveAttributes); - defaultColor = ""; -} - -export function getAttributeValue(element: Element, info: INodeInfo, attrName: string): string { - const sensitiveAttribute = attributeMaskList.indexOf(attrName) > -1; - const maskAttribute = sensitiveAttribute && !info.unmask; - const attrValue = element.attributes[attrName].value; - return maskAttribute ? mask(attrValue) : attrValue; -} - -export function getElementValue(element: Element, info: INodeInfo): string { - const valueStr = (element as any).value; - return info.unmask ? valueStr : mask(valueStr); -} - -function getLayout(element: Element): ILayoutRectangle { - let layout: ILayoutRectangle = null; - // In IE, calling getBoundingClientRect on a node that is disconnected - // from a DOM tree, sometimes results in a 'Unspecified Error' - // Wrapping this in try/catch is faster than checking whether element is connected to DOM - let rect = null; - let doc = document.documentElement; - try { - rect = element.getBoundingClientRect(); - } catch (e) { - // Ignore - } - - if (rect) { - // getBoundingClientRect returns relative positioning to viewport and therefore needs - // addition of window scroll position to get position relative to document - // Also: using Math.floor() instead of Math.round() below because in Edge, - // getBoundingClientRect returns partial pixel values (e.g. 162.5px) and Chrome already - // floors the value (e.g. 162px). Keeping behavior consistent across - layout = { - x: Math.floor(rect.left) + ("pageXOffset" in window ? window.pageXOffset : doc.scrollLeft), - y: Math.floor(rect.top) + ("pageYOffset" in window ? window.pageYOffset : doc.scrollTop), - width: Math.round(rect.width), - height: Math.round(rect.height) - }; - } - return layout; -} - -function getAttributes(element: Element, info: INodeInfo): IAttributes { - let elementAttributes = element.attributes; - let stateAttributes: IAttributes = {}; - - for (let i = 0; i < elementAttributes.length; i++) { - let attrName = elementAttributes[i].name; - stateAttributes[attrName] = getAttributeValue(element, info, attrName); - } - - return stateAttributes; -} - -function getStyles(element: Element): ILayoutStyle { - let computed = window.getComputedStyle(element); - let style = {}; - - if (defaultColor.length === 0) { - defaultColor = computed[Styles.Color]; - } - - // Send computed styles, if relevant, back to server - if (match(computed[Styles.Visibility], ["hidden", "collapse"])) { - style[Styles.Visibility] = computed[Styles.Visibility]; - } - - if (match(computed[Styles.OverflowX], ["auto", "scroll", "hidden"])) { - style[Styles.OverflowX] = computed[Styles.OverflowX]; - } - - if (match(computed[Styles.OverflowY], ["auto", "scroll", "hidden"])) { - style[Styles.OverflowY] = computed[Styles.OverflowY]; - } - - if (computed[Styles.BackgroundImage] !== "none") { - style[Styles.BackgroundImage] = computed[Styles.BackgroundImage]; - } - - if (!match(computed[Styles.BackgroundColor], ["rgba(0, 0, 0, 0)", "transparent"])) { - style[Styles.BackgroundColor] = computed[Styles.BackgroundColor]; - } - - if (computed[Styles.Color] !== defaultColor) { - style[Styles.Color] = computed[Styles.Color]; - } - - return Object.keys(style).length > 0 ? style : null; -} - -function match(variable: string, values: string[]): boolean { - return values.indexOf(variable) > -1; -} diff --git a/src/plugins/layout/states/generic.ts b/src/plugins/layout/states/generic.ts deleted file mode 100644 index 0ead2e7f..00000000 --- a/src/plugins/layout/states/generic.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { ILayoutState } from "@clarity-types/layout"; - -export const NodeIndex = "clarity-index"; - -export enum Tags { - Meta = "META", - Script = "SCRIPT", - Doc = "*DOC*", - Text = "*TXT*", - Ignore = "*IGNORE*" -} - -export function getNodeIndex(node: Node): number { - return (node && NodeIndex in node) ? node[NodeIndex] : null; -} - -export function createGenericLayoutState(node: Node, tag: string): ILayoutState { - let layoutIndex = getNodeIndex(node); - let state: ILayoutState = { - index: layoutIndex, - parent: getNodeIndex(node.parentNode), - previous: getNodeIndex(node.previousSibling), - next: getNodeIndex(node.nextSibling), - source: null, - action: null, - tag - }; - return state; -} diff --git a/src/plugins/layout/states/ignore.ts b/src/plugins/layout/states/ignore.ts deleted file mode 100644 index 6a601293..00000000 --- a/src/plugins/layout/states/ignore.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { IIgnoreLayoutState, INodeInfo } from "@clarity-types/layout"; -import { createGenericLayoutState, Tags } from "./generic"; - -export function createIgnoreLayoutState(node: Node): IIgnoreLayoutState { - let layoutState = createGenericLayoutState(node, Tags.Ignore) as IIgnoreLayoutState; - layoutState.nodeType = node.nodeType; - if (node.nodeType === Node.ELEMENT_NODE) { - layoutState.elementTag = (node as Element).tagName; - } - return layoutState; -} - -export function shouldIgnoreNode(node: Node, parentInfo: INodeInfo): boolean { - if (parentInfo && parentInfo.ignore) { - return true; - } - - let ignore = false; - switch (node.nodeType) { - case Node.ELEMENT_NODE: - let tagName = (node as Element).tagName; - if (tagName === Tags.Script || tagName === Tags.Meta) { - ignore = true; - } - break; - case Node.COMMENT_NODE: - ignore = true; - break; - case Node.TEXT_NODE: - // If we capture CSSRules on style elements, ignore its text children nodes - // Since iterating over css rules can be 100X slower on certain browsers, - // we limit ignoring text nodes to STYLE elements with empty text content - if (parentInfo && parentInfo.captureCssRules) { - ignore = true; - } - break; - default: - break; - } - return ignore; -} diff --git a/src/plugins/layout/states/stateprovider.ts b/src/plugins/layout/states/stateprovider.ts deleted file mode 100644 index ecd69e31..00000000 --- a/src/plugins/layout/states/stateprovider.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { ILayoutState, INodeInfo } from "@clarity-types/layout"; -import { createDoctypeLayoutState } from "./doctype"; -import { createElementLayoutState, resetElementStateProvider } from "./element"; -import { createIgnoreLayoutState } from "./ignore"; -import { createStyleLayoutState } from "./style"; -import { createTextLayoutState } from "./text"; - -export function createLayoutState(node: Node, info: INodeInfo): ILayoutState { - if (info.ignore) { - return createIgnoreLayoutState(node); - } - let state: ILayoutState = null; - switch (node.nodeType) { - case Node.DOCUMENT_TYPE_NODE: - state = createDoctypeLayoutState(node as DocumentType); - break; - case Node.TEXT_NODE: - state = createTextLayoutState(node as Text, info); - break; - case Node.ELEMENT_NODE: - let elem = node as Element; - switch (elem.tagName) { - case "STYLE": - state = createStyleLayoutState(elem as HTMLStyleElement, info); - break; - default: - state = createElementLayoutState(elem, info); - break; - } - break; - default: - state = createIgnoreLayoutState(node); - break; - } - return state; -} - -export function resetStateProvider(): void { - resetElementStateProvider(); -} diff --git a/src/plugins/layout/states/style.ts b/src/plugins/layout/states/style.ts deleted file mode 100644 index f7f9db56..00000000 --- a/src/plugins/layout/states/style.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { INodeInfo, IStyleLayoutState } from "@clarity-types/layout"; -import { config } from "@src/config"; -import { createElementLayoutState } from "./element"; - -export function createStyleLayoutState(styleNode: HTMLStyleElement, info: INodeInfo): IStyleLayoutState { - let layoutState = createElementLayoutState(styleNode, info) as IStyleLayoutState; - if (info.captureCssRules) { - layoutState.cssRules = getCssRules(styleNode); - } - return layoutState; -} - -export function getCssRules(element: HTMLStyleElement): string[] { - let cssRules = null; - let rules = []; - // Firefox throws a SecurityError when trying to access cssRules of a stylesheet from a different domain - try { - let sheet = element.sheet as CSSStyleSheet; - cssRules = sheet ? sheet.cssRules : []; - } catch (e) { - if (e.name !== "SecurityError") { - throw e; - } - } - - if (cssRules !== null) { - rules = []; - for (let i = 0; i < cssRules.length; i++) { - rules.push(cssRules[i].cssText); - } - } - - return rules; -} - -export function shouldCaptureCssRules(node: Node): boolean { - let captureCssRules = false; - if (node.nodeType === Node.ELEMENT_NODE && (node as Element).tagName === "STYLE") { - // Capturing CSS rules associated with a style node is more precise than relying on its childlist CSS text, - // because rules can be added/modified without affecting child text nodes, but it is also costly for performance. - // As a compromise, we are going to capture CSS rules in two cases: - // 1. If the config explicitly tells us to do so - // 2. If style node has no children. A rather common technique is to control page styles by inserting rules - // directly into the 'sheet holder' style node. So, when we encounter a style node with no inner text, - // chances are high that there are non-child-text rules associated with it and if we don't capture them, - // page visualization is likely to be broken. A popular example library that uses such technique is 'styled-components'. - // https://www.npmjs.com/package/styled-components - captureCssRules = config.cssRules || node.textContent.length === 0; - } - return captureCssRules; -} diff --git a/src/plugins/layout/states/text.ts b/src/plugins/layout/states/text.ts deleted file mode 100644 index 68074e88..00000000 --- a/src/plugins/layout/states/text.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { INodeInfo, ITextLayoutState } from "@clarity-types/layout"; -import { mask } from "@src/utils"; -import { createGenericLayoutState, Tags } from "./generic"; - -export function createTextLayoutState(textNode: Text, info: INodeInfo): ITextLayoutState { - const textState = createGenericLayoutState(textNode, Tags.Text) as ITextLayoutState; - - // Text nodes that are children of the STYLE elements contain CSS code, so we need to unmask it - const unmask = info.isCss || info.unmask; - textState.content = unmask ? textNode.nodeValue : mask(textNode.nodeValue); - return textState; -} - -export function isCssText(node: Node): boolean { - let isCss = false; - if (node.nodeType === Node.TEXT_NODE) { - // Checking parentNode, instead of parentElement, because in IE textNode.parentElement returns 'undefined'. - let parent = node.parentNode; - isCss = parent && parent.nodeType === Node.ELEMENT_NODE && (parent as Element).tagName === "STYLE"; - } - return isCss; -} diff --git a/src/plugins/performance.ts b/src/plugins/performance.ts deleted file mode 100644 index d1a51f55..00000000 --- a/src/plugins/performance.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { IPlugin } from "@clarity-types/core"; -import { IPerformanceResourceTimingState, IPerformanceTimingState } from "@clarity-types/performance"; -import { config } from "@src/config"; -import { addEvent } from "@src/core"; -import { mapProperties } from "@src/utils"; - -export const NavigationTimingEventType = "NavigationTiming"; -export const ResourceTimingEventType = "ResourceTiming"; -export const PerformanceStateErrorEventType = "PerformanceStateError"; - -export default class PerformanceProfiler implements IPlugin { - - private readonly timeoutLength: number = 1000; - - private urlBlacklist: string[]; - private lastInspectedEntryIndex: number; - private logTimingTimeout: number; - private logResourceTimingTimeout: number; - - // For performance reasons this module relies on the fact that array returned from the - // getEntries function is a superset of the array returned from that function on previous inspection. - // If that is not the case (e.g. performance.clearResourceTimings() was used), then - // some entries may be missed and consistency is not guaranteed. - // Set this flag to 'true' if we detect that something went wrong (e.g. new array length < lastInspectedEntryIndex) - private stateError: boolean = false; - - // IE has a very 'unique' way of working with performance entries - // Each time you invoke getEntries() it returns a list of brand new objects, - // even if those objects describe the same network entries, but at least it maintains the order. - // Besides that, IE returns performance entries for resources that are not finished loading yet, - // so duration, responseEnd and other values can be '0' until you re-inspect getEntries() later. - // To be able to come back to those incomplete entries, we will store their indices for revisiting. - private incompleteEntryIndices: number[] = []; - - private timing: PerformanceTiming; - private getEntriesByType: (type: string) => PerformanceEntry[]; - - public activate(): void { - if (this.timing) { - this.logTimingTimeout = window.setTimeout(this.logTiming.bind(this), this.timeoutLength); - } - if (this.getEntriesByType) { - this.logResourceTimingTimeout = window.setTimeout(this.logResourceTiming.bind(this), this.timeoutLength); - } - } - - public reset(): void { - this.lastInspectedEntryIndex = -1; - this.stateError = false; - this.incompleteEntryIndices = []; - this.urlBlacklist = config.urlBlacklist.map(this.getFullUrl); - - // We need to blacklist our own uploadUrl, otherwise we get into an infinite loop of instrumenting the calls we make - // to the uploadUrl. Each of our instrumentation calls gets instrumented and we ended up POSTing multiple times per second. - this.urlBlacklist.push(this.getFullUrl(config.uploadUrl)); - - // Potentially these don't need resets because performance object doesn't normally change within the page - // The reason for resetting these values on each activation is for easier testing - // This way this component would pick up a dummy performance object for reliable unit testing - this.timing = window.performance && performance.timing; - this.getEntriesByType = window.performance - && typeof performance.getEntriesByType === "function" - && performance.getEntriesByType.bind(performance); - } - - public teardown(): void { - clearTimeout(this.logTimingTimeout); - clearTimeout(this.logResourceTimingTimeout); - } - - private logTiming(): void { - if (this.timing.loadEventEnd > 0) { - let formattedTiming = this.timing.toJSON ? this.timing.toJSON() : this.timing; - formattedTiming = mapProperties(formattedTiming, (name: string) => { - return (formattedTiming[name] === 0) ? 0 : Math.round(formattedTiming[name] - formattedTiming.navigationStart); - }, false); - let navigationTimingEventState: IPerformanceTimingState = { - timing: formattedTiming - }; - addEvent({type: NavigationTimingEventType, state: navigationTimingEventState}); - } else { - this.logTimingTimeout = window.setTimeout(this.logTiming.bind(this), this.timeoutLength); - } - } - - private logResourceTiming(): void { - let entries = this.getEntriesByType("resource") as PerformanceResourceTiming[]; - - // If entries.length < lastInspectedEntry + 1, most likely some entries have - // been cleared, which would cause inconsistencies in further entries inspection. - // In case this happens, log an error once and reset the module to its initial state - if (entries.length < this.lastInspectedEntryIndex + 1) { - if (!this.stateError) { - this.stateError = true; - addEvent({type: PerformanceStateErrorEventType, state: {}}); - } - - this.lastInspectedEntryIndex = -1; - this.incompleteEntryIndices = []; - } - - let incompleteEntryIndicesCopy = this.incompleteEntryIndices.slice(); - - this.incompleteEntryIndices = []; - - // First, revisit the entries that were identified as incomplete upon last inspection - for (let i = 0; i < incompleteEntryIndicesCopy.length; i++) { - let entryIndex = incompleteEntryIndicesCopy[i]; - let networkData = this.inspectEntry(entries[entryIndex], entryIndex); - if (networkData) { - addEvent({type: ResourceTimingEventType, state: networkData}); - } - } - - // Then, inspect fresh entries - for (let i = this.lastInspectedEntryIndex + 1; i < entries.length; i++) { - let networkData = this.inspectEntry(entries[i], i); - if (networkData) { - addEvent({type: ResourceTimingEventType, state: networkData}); - } - this.lastInspectedEntryIndex = i; - } - - this.logResourceTimingTimeout = window.setTimeout(this.logResourceTiming.bind(this), this.timeoutLength); - } - - private inspectEntry(entry: PerformanceResourceTiming, entryIndex: number): IPerformanceResourceTimingState { - let networkData: IPerformanceResourceTimingState = null; - if (entry && entry.responseEnd > 0) { - if (this.urlBlacklist.indexOf(entry.name) < 0) { - networkData = { - duration: entry.duration, - initiatorType: entry.initiatorType, - startTime: entry.startTime, - connectStart: entry.connectStart, - connectEnd: entry.connectEnd, - requestStart: entry.requestStart, - responseStart: entry.responseStart, - responseEnd: entry.responseEnd, - name: entry.name - }; - - // These properties only exist in new browsers - // They also don't need rounding, because they are measured in bytes - if ("transferSize" in entry) { - networkData.transferSize = entry.transferSize; - } - if ("encodedBodySize" in entry) { - networkData.encodedBodySize = entry.encodedBodySize; - } - if ("decodedBodySize" in entry) { - networkData.decodedBodySize = entry.decodedBodySize; - } - if ("nextHopProtocol" in entry) { - networkData.protocol = entry.nextHopProtocol; - } - - networkData = mapProperties(networkData, (name: string, value: any) => { - return (typeof value === "number") ? Math.round(value) : value; - }, true) as IPerformanceResourceTimingState; - } - } else { - this.incompleteEntryIndices.push(entryIndex); - } - - return networkData; - } - - private getFullUrl(partialUrl: string): string { - let dummyLink = document.createElement("a"); - dummyLink.href = partialUrl; - return dummyLink.href; - } -} diff --git a/src/plugins/pointer/mouse.ts b/src/plugins/pointer/mouse.ts deleted file mode 100644 index c50aece9..00000000 --- a/src/plugins/pointer/mouse.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { IPointerState } from "@clarity-types/pointer"; -import { NodeIndex } from "@src/plugins/layout/states/generic"; - -// Accessing any evt property can sometimes (rarely) throw exception "Permission denied to access property..." -// Not adding try/catch by design for perf reasons -export function transform(evt: MouseEvent): IPointerState[] { - let de = document.documentElement; - return [{ - index: 1, /* Pointer ID */ - event: evt.type, - pointer: "mouse", - x: "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null), - y: "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null), - width: 1, - height: 1, - pressure: 1, - tiltX: 0, - tiltY: 0, - target: (evt.target && NodeIndex in evt.target) ? evt.target[NodeIndex] : null, - buttons: evt.buttons - }]; -} diff --git a/src/plugins/pointer/pointer.ts b/src/plugins/pointer/pointer.ts deleted file mode 100644 index 115af2bb..00000000 --- a/src/plugins/pointer/pointer.ts +++ /dev/null @@ -1,70 +0,0 @@ -import * as mouse from "./mouse"; -import * as touch from "./touch"; - -import { IPlugin } from "@clarity-types/core"; -import { IPointerModule, IPointerState } from "@clarity-types/pointer"; -import { addEvent, bind } from "@src/core"; - -export default class Pointer implements IPlugin { - private eventName: string = "Pointer"; - private distanceThreshold: number = 20; - private timeThreshold: number = 500; - private lastMoveState: IPointerState; - private lastMoveTime: number; - - public activate(): void { - bind(document, "mousedown", this.pointerHandler.bind(this, mouse)); - bind(document, "mouseup", this.pointerHandler.bind(this, mouse)); - bind(document, "mousemove", this.pointerHandler.bind(this, mouse)); - bind(document, "mousewheel", this.pointerHandler.bind(this, mouse)); - bind(document, "click", this.pointerHandler.bind(this, mouse)); - bind(document, "touchstart", this.pointerHandler.bind(this, touch)); - bind(document, "touchend", this.pointerHandler.bind(this, touch)); - bind(document, "touchmove", this.pointerHandler.bind(this, touch)); - bind(document, "touchcancel", this.pointerHandler.bind(this, touch)); - } - - public teardown(): void { - // Nothing to teardown - } - - public reset(): void { - this.lastMoveState = null; - this.lastMoveTime = 0; - } - - private pointerHandler(handler: IPointerModule, evt: Event): void { - let states = handler.transform(evt); - for (let state of states) { - this.processState(state, evt.timeStamp); - } - } - - private processState(state: IPointerState, time: number): void { - switch (state.event) { - case "mousemove": - case "touchmove": - if (this.lastMoveState == null - || this.checkDistance(this.lastMoveState, state) - || this.checkTime(time)) { - this.lastMoveState = state; - this.lastMoveTime = time; - addEvent({type: this.eventName, state}); - } - break; - default: - addEvent({type: this.eventName, state}); - break; - } - } - - private checkDistance(stateOne: IPointerState, stateTwo: IPointerState): boolean { - let dx = stateOne.x - stateTwo.x; - let dy = stateOne.y - stateTwo.y; - return (dx * dx + dy * dy > this.distanceThreshold * this.distanceThreshold); - } - - private checkTime(time: number): boolean { - return time - this.lastMoveTime > this.timeThreshold; - } -} diff --git a/src/plugins/pointer/touch.ts b/src/plugins/pointer/touch.ts deleted file mode 100644 index f3e32f98..00000000 --- a/src/plugins/pointer/touch.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { IPointerState } from "@clarity-types/pointer"; -import { NodeIndex } from "@src/plugins/layout/states/generic"; - -// Accessing any evt property can sometimes (rarely) throw exception "Permission denied to access property..." -// Not adding try/catch by design for perf reasons -export function transform(evt: TouchEvent): IPointerState[] { - let states: IPointerState[] = []; - let de = document.documentElement; - let buttons = (evt.type === "touchstart" || evt.type === "touchmove") ? 1 : 0; - let touches = evt.changedTouches; - if (touches) { - for (let i = 0; i < touches.length; i++) { - let touch = touches[i]; - states.push({ - index: touch.identifier + 2, /* Avoid conflict with mouse index of 1 */ - event: evt.type, - pointer: "touch", - x: "clientX" in touch ? Math.round(touch.clientX + de.scrollLeft) : null, - y: "clientY" in touch ? Math.round(touch.clientY + de.scrollTop) : null, - width: "radiusX" in touch ? Math.round(touch["radiusX"]) : ("webkitRadiusX" in touch ? Math.round(touch["webkitRadiusX"]) : 0), - height: "radiusY" in touch ? Math.round(touch["radiusY"]) : ("webkitRadiusY" in touch ? Math.round(touch["webkitRadiusY"]) : 0), - pressure: "force" in touch ? touch["force"] : ("webkitForce" in touch ? touch["webkitForce"] : 0.5), - tiltX: 0, - tiltY: 0, - target: (evt.target && NodeIndex in evt.target) ? evt.target[NodeIndex] : null, - buttons - }); - } - } - return states; -} diff --git a/src/plugins/viewport.ts b/src/plugins/viewport.ts deleted file mode 100644 index bd89c274..00000000 --- a/src/plugins/viewport.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { IPlugin } from "@clarity-types/core"; -import { IViewportState } from "@clarity-types/viewport"; -import { addEvent, bind } from "@src/core"; - -export default class Viewport implements IPlugin { - private eventName: string = "Viewport"; - private distanceThreshold: number = 20; - private lastViewportState: IViewportState; - private body: HTMLElement = document.body; - private documentElement: HTMLElement = document.documentElement; - - public activate(): void { - this.processState(this.getViewport("discover")); - bind(window, "scroll", this.viewportHandler.bind(this)); - bind(window, "resize", this.viewportHandler.bind(this)); - bind(window, "pageshow", this.viewportHandler.bind(this)); - bind(window, "pagehide", this.viewportHandler.bind(this)); - bind(document, "visibilitychange", this.viewportHandler.bind(this)); - } - - public teardown(): void { - // Nothing to teardown - } - - public reset(): void { - this.lastViewportState = null; - } - - private viewportHandler(evt: Event): void { - let viewportState = this.getViewport(evt.type); - this.processState(viewportState); - } - - private getViewport(type: string): IViewportState { - let viewport: IViewportState = { - viewport: { - x: "pageXOffset" in window ? window.pageXOffset : this.documentElement.scrollLeft, - y: "pageYOffset" in window ? window.pageYOffset : this.documentElement.scrollTop, - width: "innerWidth" in window ? window.innerWidth : this.documentElement.clientWidth, - height: "innerHeight" in window ? window.innerHeight : this.documentElement.clientHeight - }, - document: { - width: this.body ? this.body.clientWidth : null, - height: this.getDocumentHeight() - }, - dpi: "devicePixelRatio" in window ? window.devicePixelRatio : -1, - visibility: "visibilityState" in document ? document.visibilityState : "default", - event: type - }; - return viewport; - } - - // body.clientHeight gets set to viewport height when doctype is not set for a document. - // The more accurate way to calculate browser height is to get the maximum of body and documentElement heights - private getDocumentHeight(): number { - let bodyClientHeight = this.body ? this.body.clientHeight : null; - let bodyScrollHeight = this.body ? this.body.scrollHeight : null; - let bodyOffsetHeight = this.body ? this.body.offsetHeight : null; - let documentClientHeight = this.documentElement ? this.documentElement.clientHeight : null; - let documentScrollHeight = this.documentElement ? this.documentElement.scrollHeight : null; - let documentOffsetHeight = this.documentElement ? this.documentElement.offsetHeight : null; - let documentHeight = Math.max(bodyClientHeight, bodyScrollHeight, bodyOffsetHeight, - documentClientHeight, documentScrollHeight, documentOffsetHeight); - return documentHeight; - } - - private processState(state: IViewportState): void { - let recordState = true; - if (state.event === "scroll" - && this.lastViewportState !== null - && !this.checkDistance(this.lastViewportState, state)) { - recordState = false; - } - if (recordState) { - this.lastViewportState = state; - addEvent({type: this.eventName, state}); - } - } - - private checkDistance(stateOne: IViewportState, stateTwo: IViewportState): boolean { - let dx = stateOne.viewport.x - stateTwo.viewport.x; - let dy = stateOne.viewport.y - stateTwo.viewport.y; - return (dx * dx + dy * dy > this.distanceThreshold * this.distanceThreshold); - } -} diff --git a/src/upload.ts b/src/upload.ts deleted file mode 100644 index 696dc87e..00000000 --- a/src/upload.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { IPayload, IPayloadInfo, State, UploadCallback } from "@clarity-types/core"; -import { Instrumentation, ITotalByteLimitExceededEventState, IXhrErrorEventState } from "@clarity-types/instrumentation"; -import { config } from "./config"; -import { ClarityAttribute, instrument, state, teardown } from "./core"; -import { debug, getEventId } from "./utils"; - -// Counters -let sentBytesCount: number; -let uploadCount: number; - -// Storage for payloads that are compressed and are ready to be sent, but are waiting for Clarity trigger. -// Once trigger is fired, all payloads from this queue will be sent in the order they were generated. -let payloadQueue: IPayloadInfo[]; - -// Map from a sequence number to an UploadInfo object for payloads that were attempted to be sent, but were not succesfully delivered yet -let payloadInfos: { [key: number]: IPayloadInfo }; - -let reUploadQueue: number[]; - -export function upload(compressed: string, raw: IPayload, onSuccessCustom?: UploadCallback, onFailureCustom?: UploadCallback): void { - let uploadHandler = config.uploadHandler || defaultUploadHandler; - let sequenceNo = raw.envelope.sequenceNumber; - let onSuccess = getOnUploadCompletedHandler(true, sequenceNo, onSuccessCustom); - let onFailure = getOnUploadCompletedHandler(false, sequenceNo, onFailureCustom); - payloadInfos[sequenceNo] = { compressed, raw, failureCount: 0 }; - uploadHandler(compressed, onSuccess, onFailure); - - if (config.debug) { - let rawLength = JSON.stringify(raw).length; - debug(`** Upload #${uploadCount}: Batch #${sequenceNo}` - + ` ${Math.round(compressed.length / 1024.0)}KB` - + ` (${Math.round(rawLength / 1024.0)}KB Raw). **`); - } - uploadCount++; -} - -export function enqueuePayload(compressed: string, raw: IPayload): void { - let payloadInfo: IPayloadInfo = { compressed, raw, failureCount: 0 }; - payloadQueue.push(payloadInfo); -} - -export function flushPayloadQueue(): void { - uploadMultiplePayloads(payloadQueue); -} - -export function resetUploads(): void { - sentBytesCount = 0; - uploadCount = 0; - payloadQueue = []; - payloadInfos = {}; - reUploadQueue = []; -} - -function uploadMultiplePayloads(payloads: IPayloadInfo[]): void { - if (payloads.length > 0) { - let nextPayload = payloads.shift(); - let uploadNextPayload = (): void => { - uploadMultiplePayloads(payloads); - }; - upload(nextPayload.compressed, nextPayload.raw, uploadNextPayload, uploadNextPayload); - } -} - -function defaultUploadHandler(payload: string, onSuccess?: UploadCallback, onFailure?: UploadCallback): void { - if (config.uploadUrl.length > 0) { - payload = JSON.stringify(payload); - let xhr = new XMLHttpRequest(); - let headers = Object.keys(config.uploadHeaders); - xhr.open("POST", config.uploadUrl); - for (let i = 0; i < headers.length; i++) { - xhr.setRequestHeader(headers[i], config.uploadHeaders[headers[i]]); - } - xhr.onreadystatechange = (): void => { - onXhrReadyStatusChange(xhr, onSuccess, onFailure); - }; - xhr.send(payload); - } -} - -function onXhrReadyStatusChange(xhr: XMLHttpRequest, onSuccess: UploadCallback, onFailure: UploadCallback): void { - if (xhr.readyState === XMLHttpRequest.DONE) { - // HTTP response status documentation: - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status - if (xhr.status < 200 || xhr.status > 208) { - onFailure(xhr.status); - } else { - onSuccess(xhr.status); - } - } -} - -function onSuccessDefault(sequenceNumber: number): void { - debug(`SUCCESS: Delivered batch #${sequenceNumber}`); - let sentData = payloadInfos[sequenceNumber]; - delete payloadInfos[sequenceNumber]; - sentBytesCount += sentData.compressed.length; - - if (state === State.Activated && sentBytesCount > config.totalLimit) { - let totalByteLimitExceededEventState: ITotalByteLimitExceededEventState = { - type: Instrumentation.TotalByteLimitExceeded, - bytes: sentBytesCount - }; - instrument(totalByteLimitExceededEventState); - teardown(); - return; - } - - retryFailedUploads(); -} - -function onFailureDefault(sequenceNumber: number, status: number): void { - debug(`FAILED: Delivery failed for batch #${sequenceNumber} Status: ${status}`); - let uploadInfo = payloadInfos[sequenceNumber]; - uploadInfo.failureCount++; - logXhrError(sequenceNumber, status, uploadInfo); - if (uploadInfo.failureCount <= config.reUploadLimit) { - reUploadQueue.push(sequenceNumber); - } -} - -function logXhrError(sequenceNumber: number, status: number, payloadInfo: IPayloadInfo): void { - let events = payloadInfo.raw.events; - let xhrErrorState: IXhrErrorEventState = { - type: Instrumentation.XhrError, - requestStatus: status, - sequenceNumber, - compressedLength: payloadInfo.compressed.length, - rawLength: JSON.stringify(payloadInfo.raw).length, - firstEventId: getEventId(events[0]), - lastEventId: getEventId(events[events.length - 1]), - attemptNumber: payloadInfo.failureCount - }; - instrument(xhrErrorState); -} - -function retryFailedUploads(): void { - let payloads: IPayloadInfo[] = []; - while (reUploadQueue.length > 0) { - let nextPayloadSequenceNo = reUploadQueue.shift(); - payloads.push(payloadInfos[nextPayloadSequenceNo]); - } - uploadMultiplePayloads(payloads); -} - -function getOnUploadCompletedHandler(success: boolean, sequenceNo: number, customHandler: UploadCallback): UploadCallback { - let defaultHandler = success ? onSuccessDefault : onFailureDefault; - let sourceImpressionId = document[ClarityAttribute]; - return (status: number): void => { - let currentImpressionId = document[ClarityAttribute]; - if (state === State.Activated && currentImpressionId === sourceImpressionId) { - defaultHandler(sequenceNo, status); - if (customHandler) { - customHandler(status); - } - } - }; -} diff --git a/src/utils.ts b/src/utils.ts deleted file mode 100644 index 6112d2e8..00000000 --- a/src/utils.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { IEventArray } from "@clarity-types/core"; -import { IClarityAssertFailedEventState, Instrumentation } from "@clarity-types/instrumentation"; -import { config } from "./config"; -import { instrument } from "./core"; - -// Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript -// Excluding 3rd party code from tslint -// tslint:disable -export function guid() { - let d = new Date().getTime(); - if (window.performance && performance.now) { - // Use high-precision timer if available - d += performance.now(); - } - let uuid = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, function(c) { - let r = (d + Math.random() * 16) % 16 | 0; - d = Math.floor(d / 16); - return (c == "x" ? r : (r & 0x3 | 0x8)).toString(16); - }); - return uuid; -} -// tslint:enable - -export function setCookie(cookieName: string, value: string, expDays?: number): void { - let expDate = null; - if (expDays) { - expDate = new Date(); - expDate.setDate(expDate.getDate() + expDays); - } - let expires = expDate ? "expires=" + expDate.toUTCString() : ""; - let cookieValue = value + ";" + expires + ";path=/"; - document.cookie = cookieName + "=" + cookieValue; -} - -export function getCookie(cookieName: string): string { - let arrayOfCookies: string[] = document.cookie.split(";"); - if (arrayOfCookies) { - for (let i = 0; i < arrayOfCookies.length; i++) { - let cookiePair: string[] = arrayOfCookies[i].split("="); - if (cookiePair && cookiePair.length > 1 && cookiePair[0].indexOf(cookieName) >= 0) { - return cookiePair[1]; - } - } - } - return null; -} - -export function mapProperties(sourceObj: object, - mapFunction: (name: string, value: any) => any, - ownPropertiesOnly: boolean, - outObj?: object): object { - outObj = outObj || {}; - for (let property in sourceObj) { - if (!ownPropertiesOnly || sourceObj.hasOwnProperty(property)) { - let sourceValue = sourceObj[property]; - let outValue = mapFunction ? mapFunction(property, sourceValue) : sourceValue; - if (typeof outValue !== "undefined") { - outObj[property] = outValue; - } - } - } - return outObj; -} - -export function traverseNodeTree(root: Node, processingFunc: (node: Node) => void, includeRoot: boolean = true): void { - let queue = [root]; - while (queue.length > 0) { - let next = queue.shift(); - let nextChild: Node = next.firstChild; - while (nextChild) { - queue.push(nextChild); - nextChild = nextChild.nextSibling; - } - if (next !== root || includeRoot) { - processingFunc(next); - } - } -} - -export function roundingMapFunction(name: string, value: any): any { - return (isNumber(value)) ? Math.round(value) : value; -} - -export function isNumber(value: any): boolean { - return (typeof value === "number" && !isNaN(value)); -} - -export function assert(condition: boolean, source: string, comment: string): void { - if (condition === false) { - debug(`>>> Clarity Assert failed\nSource: ${source}\nComment: ${comment}`); - let eventState: IClarityAssertFailedEventState = { - type: Instrumentation.ClarityAssertFailed, - source, - comment - }; - instrument(eventState); - } -} - -export function debug(text: string): void { - if (config.debug && console.log) { - console.log(`(Clarity) ${text}`); - } -} - -export function getEventId(eventArray: IEventArray): number { - return eventArray[0]; -} - -export function getEventType(eventArray: IEventArray): string { - return eventArray[1]; -} - -export function mask(text: string): string { - return text.replace(/\S/gi, "*"); -} diff --git a/types/compressionworker.d.ts b/types/compressionworker.d.ts deleted file mode 100644 index bc30074b..00000000 --- a/types/compressionworker.d.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { IEventArray, IPayload } from "./core"; - -export const enum WorkerMessageType { - /* Main thread to Worker messages */ - AddEvent, - ForceCompression, - - /* Worker to main thread messages */ - CompressedBatch -} - -export interface IWorkerMessage { - type: WorkerMessageType; -} - -export interface ITimestampedWorkerMessage extends IWorkerMessage { - time: number; -} - -export interface IAddEventMessage extends ITimestampedWorkerMessage { - event: IEventArray; - isXhrErrorEvent?: boolean; -} - -export interface ICompressedBatchMessage extends IWorkerMessage { - compressedData: string; - rawData: IPayload; -} diff --git a/types/config.d.ts b/types/config.d.ts deleted file mode 100644 index 160e562b..00000000 --- a/types/config.d.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { IClarityFields, UploadHandler } from "./core"; - -export interface IConfig { - // Active plugins - plugins?: string[]; - - // Endpoint, to which data will be uploaded - uploadUrl?: string; - - // A list of URLs to ignore when instrumenting network resource entries - // This is useful to prevent Clarity from instrumenting its own activity - urlBlacklist?: string[]; - - // Each new event is going to delay data upload to server by this number of milliseconds - delay?: number; - - // Maximum number of event bytes that Clarity can send in a single upload - batchLimit?: number; - - // Maximum number of bytes that Clarity can send per page overall - totalLimit?: number; - - // Maximum number of XHR re-delivery attempts for a single payload - reUploadLimit?: number; - - // If set to 'true', clarity-js will NOT write CID (Clarity ID) cookie - disableCookie?: boolean; - - // Names of the attributes that need to be masked (on top of default ones), when showText is false - sensitiveAttributes?: string[]; - - // Send back instrumentation data, if set to true - instrument?: boolean; - - // Inspect CSSRuleList for style elements and send CSSRules data instead of style's text children - cssRules?: boolean; - - // Pointer to the function which would be responsible for sending the data - // If left unspecified, raw payloads will be uploaded to the uploadUrl endpoint - uploadHandler?: UploadHandler; - - // XHLHttpRequest headers to be added to every upload request with the default upload handler - // Object is a map from header names to header values - uploadHeaders?: { [key: string]: string; }; - - // Pointer to the function which will be responsible for giving Clarity - // a dictionary of strings that the user wants logged in each Clarity payload - customInstrumentation?: (fields: IClarityFields) => { [key: string]: string; }; - - // Setting to enable debug features (e.g. console.log statements) - debug?: boolean; - - // Setting to enable consistency verifications between real DOM and shadow DOM - // Validating consistency can be costly performance-wise, because it requires - // re-traversing entire DOM and ShadowDom to compare them against each other. - // The upside is knowing deterministically that all activity on the page was - // interpreted correctly and data is reliable. - validateConsistency?: boolean; - - // If this flag is enabled, Clarity will not send any data until trigger function is called. - // Clarity will still run in the background collecting events and compressing them into batches, - // but actual sending will only be done one the trigger is fired. - backgroundMode?: boolean; - - // Identifier of the project to which this impression will be merged on the backend - projectId?: string; -} diff --git a/types/core.d.ts b/types/core.d.ts index bd6647e2..2426230b 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -1,82 +1,2 @@ export const enum State { - Loaded, - Activated, - Unloaded, - Activating, - Unloading } - -export interface IPlugin { - activate(): void; - teardown(): void; - reset(): void; -} - -export interface IPayload { - envelope: IEnvelope; - events: IEventArray[]; -} - -export interface IPayloadInfo { - compressed: string; - raw: IPayload; - failureCount: number; -} - -export interface IEnvelope { - clarityId: string; - impressionId: string; - projectId: string; - url: string; - version: string; - time?: number; - sequenceNumber?: number; - extraInfo?: object; -} - -export interface IEventData { - type: string; - state: any; - time?: number; -} - -export interface IEvent extends IEventData { - id: number; /* Event ID */ - time: number; /* Time relative to page start */ -} - -export interface IEventBindingPair { - target: EventTarget; - listener: EventListener; -} - -export interface IBindingContainer { - [key: string]: IEventBindingPair[]; -} - -export interface IClarityFields { - impressionId: string; - clientId: string; - projectId: string; -} - -export type UploadCallback = (status: number) => void; -export type UploadHandler = (payload: string, onSuccess?: UploadCallback, onFailure?: UploadCallback) => void; - -// IEvent object converted to a value array representation -export type IEventArray = [ - number, // id - string, // type - number, // time - any[], // state - ClarityDataSchema // data schema, if it's a new one, otherwise - schema hashcode -]; - -export const enum ObjectType { - Object, - Array -} - -// For details on schema generation, see schema.md: -// https://github.com/Microsoft/clarity-js/blob/master/converters/schema.md -export type ClarityDataSchema = null | string | any[]; diff --git a/types/index.d.ts b/types/index.d.ts index 49e6bc78..8b741fb2 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -1,38 +1,5 @@ -import { IConfig } from "./config"; -import { ClarityDataSchema, IEvent, IEventArray } from "./core"; - -declare class SchemaManager { - constructor(); - public reset(): void; - public addSchema(schema: ClarityDataSchema): boolean; - public createSchema(data: any, name?: string): ClarityDataSchema; - public getSchema(schemaId: number): ClarityDataSchema; - public getSchemaId(schema: ClarityDataSchema): number | undefined; -} - interface IClarityJs { version: string; - start(config?: IConfig): void; - stop(): void; - trigger(key: string): void; -} - -interface IPayloadEncoder { - SchemaManager: typeof SchemaManager; - encode(event: IEvent): IEventArray; - decode(eventArray: IEventArray, schemas?: SchemaManager): IEvent; } declare const ClarityJs: IClarityJs; -declare const PayloadEncoder: IPayloadEncoder; - -export * from "./compressionworker"; -export * from "./config"; -export * from "./core"; -export * from "./instrumentation"; -export * from "./layout"; -export * from "./performance"; -export * from "./pointer"; -export * from "./viewport"; - -export { ClarityJs, PayloadEncoder }; diff --git a/types/instrumentation.d.ts b/types/instrumentation.d.ts deleted file mode 100644 index d60f75f0..00000000 --- a/types/instrumentation.d.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { ILayoutRoutineInfo, NumberJson } from "./layout"; - -export const enum Instrumentation { - JsError, - MissingFeature, - XhrError, - TotalByteLimitExceeded, - Teardown, - ClarityAssertFailed, - ClarityDuplicated, - ShadowDomInconsistent, - ClarityActivateError, - Trigger, - Performance -} - -export interface IInstrumentationEventState { - type: Instrumentation; -} - -export interface IJsErrorEventState extends IInstrumentationEventState { - source: string; - message: string; - stack: string; - lineno: number; - colno: number; -} - -export interface IMissingFeatureEventState extends IInstrumentationEventState { - missingFeatures: string[]; -} - -export interface IXhrErrorEventState extends IInstrumentationEventState { - requestStatus: number; - sequenceNumber: number; - compressedLength: number; - rawLength: number; - firstEventId: number; - lastEventId: number; - attemptNumber: number; -} - -export interface ITotalByteLimitExceededEventState extends IInstrumentationEventState { - bytes: number; -} - -export interface IClarityAssertFailedEventState extends IInstrumentationEventState { - source: string; - comment: string; -} - -export interface IClarityDuplicatedEventState extends IInstrumentationEventState { - currentImpressionId: string; -} - -export interface IShadowDomInconsistentEventState extends IInstrumentationEventState { - // JSON of node indicies, representing the DOM - dom: NumberJson; - - // JSON of ShadowNode IDs, representing the inconsistent ShadowDom - shadowDom: NumberJson; - - // JSON of ShadowNode IDs, representing the last consistent ShadowDom - lastConsistentShadowDom: NumberJson; - - // Last action that happened before we found out that ShadowDom is inconsistent - lastAction: ILayoutRoutineInfo; - - // To handle specific MutationObserver behavior in IE, we wait for ShadowDom to become inconsistent twice in a row, - // before we stop processing mutations and send ShadowDomInconsistentEvent. This means that the actual transition - // from consistent to inconsistent state happened on some previous action and there was also an event created for it. - // That first event is sent in this property. - firstEvent?: IShadowDomInconsistentEventState; -} - -export interface IClarityActivateErrorState extends IInstrumentationEventState { - error: string; -} - -export interface ITriggerState extends IInstrumentationEventState { - key: string; -} - -export interface IPerformanceState extends IInstrumentationEventState { - procedure: string; - duration: number; -} - -export interface INodeStateGenPerformanceState extends IPerformanceState { - nodeCount: number; -} - -export interface IMutationPerformanceState extends IPerformanceState { - mutationCount: number; - mutationSequence: number; - stateGenDuration: number; - summaryCounts: { - inserts: number; - moves: number; - updates: number; - removes: number; - }; -} diff --git a/types/layout.d.ts b/types/layout.d.ts deleted file mode 100644 index 22b43b0b..00000000 --- a/types/layout.d.ts +++ /dev/null @@ -1,133 +0,0 @@ -export type NumberJson = Array; - -export type InsertRuleHandler = (rule: string, index?: number) => number; - -export interface IShadowDomNode extends HTMLDivElement { - node: Node; - info: INodeInfo; - state: ILayoutState; - computeInfo: () => INodeInfo; - computeState: () => ILayoutState; -} - -/* Computed CSS styles associated with a layout element */ -export interface ILayoutStyle { - visibility?: string; - color?: string; - backgroundColor?: string; - backgroundImage?: string; - overflowX?: string; - overflowY?: string; -} - -export interface ILayoutRectangle { - x: number; /* X coordinate of the element */ - y: number; /* Y coordinate of the element */ - width: number; /* Width of the element */ - height: number; /* Height of the element */ - scrollX?: number; /* Scroll left of the element */ - scrollY?: number; /* Scroll top of the element */ -} - -export const enum Source { - Discover, - Mutation, - Scroll, - Input, - Css -} - -export const enum Action { - Insert, - Update, - Remove, - Move -} - -export interface IAttributes { - [key: string]: string; -} - -export interface INodeInfo { - ignore: boolean; - unmask: boolean; - isCss: boolean; - captureCssRules: boolean; -} - -export interface ILayoutState { - index: number; /* Index of the layout element */ - tag: string; /* Tag name of the element */ - source: Source; /* Source of discovery */ - action: Action; /* Reflect the action with respect to DOM */ - parent: number; /* Index of the parent element */ - previous: number; /* Index of the previous sibling, if known */ - next: number; /* Index of the next sibling, if known */ - mutationSequence?: number; /* Sequence number of the mutation batch */ -} - -export interface IDoctypeLayoutState extends ILayoutState { - attributes: { - name: string; - publicId: string; - systemId: string; - }; -} - -export interface IElementLayoutState extends ILayoutState { - attributes: IAttributes; /* Attributes associated with an element */ - layout: ILayoutRectangle; /* Layout rectangle */ - style: ILayoutStyle; /* Layout computed styles */ - value?: string; /* Applies to certain elements only https://www.w3schools.com/tags/att_value.asp + HTMLTextAreaElement */ -} - -export interface IStyleLayoutState extends IElementLayoutState { - cssRules: string[]; -} - -export interface ITextLayoutState extends ILayoutState { - content: string; -} - -export interface IIgnoreLayoutState extends ILayoutState { - nodeType: number; - elementTag?: string; -} - -export interface IMutationEntry { - node: Node; - action: Action; - parent?: Node; - previous?: Node; - next?: Node; -} - -export const enum LayoutRoutine { - DiscoverDom, - Mutation -} - -export interface ILayoutRoutineInfo { - action: LayoutRoutine; -} - -export interface IMutationRoutineInfo extends ILayoutRoutineInfo { - mutationSequence: number; /* Sequence number of the mutation batch */ - batchSize: number; /* Number of mutation records in the mutation callback */ -} - -// export interface to store some information about the initial state of the node -// in cases where mutation happens before we process the node for the first time. -// Stroring original values of the properties that can were mutated allows us to -// re-construct the initial state of the node even after mutation has happened. -export interface INodePreUpdateInfo { - characterData?: string; - attributes?: IAttributes; -} - -export interface IShadowDomMutationSummary { - newNodes: IShadowDomNode[]; - movedNodes: IShadowDomNode[]; - removedNodes: IShadowDomNode[]; - updatedNodes: IShadowDomNode[]; -} diff --git a/types/performance.d.ts b/types/performance.d.ts deleted file mode 100644 index 414e2b88..00000000 --- a/types/performance.d.ts +++ /dev/null @@ -1,22 +0,0 @@ -// We send back all properties from performance.timing object -export type IPerformanceTiming = PerformanceTiming; - -export interface IPerformanceTimingState { - timing: IPerformanceTiming; -} - -export interface IPerformanceResourceTimingState { - duration: number; - initiatorType: string; - startTime: number; - connectStart: number; - connectEnd: number; - requestStart: number; - responseStart: number; - responseEnd: number; - name: string; - transferSize?: number; - encodedBodySize?: number; - decodedBodySize?: number; - protocol?: string; -} diff --git a/types/pointer.d.ts b/types/pointer.d.ts deleted file mode 100644 index 53e3aea0..00000000 --- a/types/pointer.d.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { IEvent } from "./core"; - -export interface IPointerEvent extends IEvent { - state: IPointerState; -} - -export interface IPointerModule { - transform(evt: Event): IPointerState[]; -} - -/* Spec: https://www.w3.org/TR/pointerevents/#pointerevent-export interface */ -export interface IPointerState { - index: number; /* Pointer ID */ - event: string; /* Original event that is mapped to pointer event */ - pointer: string; /* pointerType: mouse, pen, touch */ - x: number; /* X-Axis */ - y: number; /* Y-Axis */ - width: number; - height: number; - pressure: number; - tiltX: number; - tiltY: number; - target: number; /* Layout index of the target element */ - buttons: number; -} diff --git a/types/viewport.d.ts b/types/viewport.d.ts deleted file mode 100644 index 2cc19b66..00000000 --- a/types/viewport.d.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { IEvent } from "./core"; - -export interface IViewportEvent extends IEvent { - state: IViewportState; -} - -export interface IViewportRectangle { - x: number; /* X coordinate of the element */ - y: number; /* Y coordinate of the element */ - width: number; /* Width of the element */ - height: number; /* Height of the element */ -} - -export interface IDocumentSize { - width: number; /* Document width */ - height: number; /* Document height */ -} - -export interface IViewportState { - viewport: IViewportRectangle; /* Viewport rectangle */ - document: IDocumentSize; /* Document size */ - dpi: number; /* DPI */ - visibility: string; /* Visibility state of the page */ - event: string; /* Source event */ -} From d4fb759cb67822993031e59da925349599d210f6 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 8 Apr 2019 11:31:51 -0700 Subject: [PATCH 002/105] Restructuring code and adding async support --- package.json | 2 +- src/clarity.ts | 6 +- src/core.ts | 11 +++- src/data/state.ts | 4 ++ src/dom/discover.ts | 16 +++++ src/dom/node.ts | 108 ++++++++++++++++++++++++++++++++++ src/instrument/counter.ts | 46 +++++++++++++++ src/lib/hash.ts | 12 ++++ src/lib/method.ts | 5 ++ src/lib/nodetree.ts | 120 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 327 insertions(+), 3 deletions(-) create mode 100644 src/data/state.ts create mode 100644 src/dom/discover.ts create mode 100644 src/dom/node.ts create mode 100644 src/instrument/counter.ts create mode 100644 src/lib/hash.ts create mode 100644 src/lib/method.ts create mode 100644 src/lib/nodetree.ts diff --git a/package.json b/package.json index f431eeeb..79d3886a 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "webpack-merge": "^4.2.1" }, "scripts": { - "build": "yarn build:clean && yarn build:main && yarn build:cjs", + "build": "yarn build:clean && yarn build:main && yarn build:cjs:dev", "build:main": "webpack --config webpack/configs/index.ts", "build:cjs": "webpack --config webpack/configs/prod.ts", "build:cjs:dev": "webpack --config webpack/configs/dev.ts", diff --git a/src/clarity.ts b/src/clarity.ts index 8fb3a837..2ede457d 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,3 +1,7 @@ +import {init} from "./core"; + export function start(): void { - console.log("clarity"); + init(); } + +start(); diff --git a/src/core.ts b/src/core.ts index 732be94c..e8e0bc35 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1 +1,10 @@ -let exist = true; +import discover from "./dom/discover"; + +/* Initial discovery of DOM */ +export function init(): void { + discover().then(() => { + // DEBUG: Remove later + console.log("done discovery!"); + console.log(window["TRACKER"][0]["duration"] + "ms in " + window["TRACKER"][0]["count"] + " iterations"); + }); +} diff --git a/src/data/state.ts b/src/data/state.ts new file mode 100644 index 00000000..19bb56f5 --- /dev/null +++ b/src/data/state.ts @@ -0,0 +1,4 @@ +import {NodeTree} from "../lib/nodetree"; + +export let nodes = new NodeTree(); +window["NODES"] = nodes; // DEBUG: Remove later diff --git a/src/dom/discover.ts b/src/dom/discover.ts new file mode 100644 index 00000000..4e542241 --- /dev/null +++ b/src/dom/discover.ts @@ -0,0 +1,16 @@ +import * as counter from "../instrument/counter"; +import { Method } from "../lib/method"; +import processNode from "./node"; + +export default async function(): Promise { + let method = Method.Discover; + counter.start(method); + let walker = document.createTreeWalker(document, NodeFilter.SHOW_ALL, null, false); + let node = walker.nextNode(); + while (node) { + if (counter.longtasks(method)) { await counter.idle(method); } + processNode(node); + node = walker.nextNode(); + } + counter.stop(Method.Discover); +} diff --git a/src/dom/node.ts b/src/dom/node.ts new file mode 100644 index 00000000..f9d2c6bf --- /dev/null +++ b/src/dom/node.ts @@ -0,0 +1,108 @@ +import {nodes} from "../data/state"; + +let ignoreAttributes = ["title", "alt"]; + +export default function(node: Node): void { + let parent = node.parentElement; + switch (node.nodeType) { + case Node.DOCUMENT_TYPE_NODE: + let doctype = node as DocumentType; + nodes.add(parent, node, { + attributes: { + name: doctype.name, + publicId: doctype.publicId, + systemId: doctype.systemId + } + }); + break; + case Node.TEXT_NODE: + // Account for this text node only if we are tracking the parent node + if (parent && nodes.has(parent)) { + // nodes.update(parent, {leaf: true}); + nodes.add(parent, node, { + leaf: false, + value: node.nodeValue, + layout: getTextLayout(node) + }); + } + break; + case Node.ELEMENT_NODE: + let element = ( node as HTMLElement); + switch (element.tagName) { + case "SCRIPT": + case "NOSCRIPT": + case "META": + break; + default: + nodes.add(parent, node, { + leaf: node.childNodes.length === 0, + attributes: getAttributes(element.attributes), + layout: getLayout(element) + }); + break; + } + break; + default: + break; + } +} + +function getAttributes(attributes: NamedNodeMap): {[key: string]: string} { + let output = {}; + if (attributes && attributes.length > 0) { + for (let i = 0; i < attributes.length; i++) { + let name = attributes[i].name; + if (ignoreAttributes.indexOf(name) < 0) { + output[name] = attributes[i].value; + } + } + } + return output; +} + +function getTextLayout(textNode: Node): string { + let layouts: string[] = []; + let range = document.createRange(); + range.selectNodeContents(textNode); + let rects = range.getClientRects(); + let doc = document.documentElement; + for (let i = 0; i < rects.length; i++) { + let rect = rects[i]; + layouts.push([ + Math.floor(rect.left) + ("pageXOffset" in window ? window.pageXOffset : doc.scrollLeft), + Math.floor(rect.top) + ("pageYOffset" in window ? window.pageYOffset : doc.scrollTop), + Math.round(rect.width), + Math.round(rect.height) + ].join("x")); + } + return layouts.join("."); +} + +function getLayout(element: Element): string { + // In IE, calling getBoundingClientRect on a node that is disconnected + // from a DOM tree, sometimes results in a 'Unspecified Error' + // Wrapping this in try/catch is faster than checking whether element is connected to DOM + let layout: number[] = []; + let rect = null; + let doc = document.documentElement; + try { + rect = element.getBoundingClientRect(); + } catch (e) { + // Ignore + } + + if (rect) { + // getBoundingClientRect returns relative positioning to viewport and therefore needs + // addition of window scroll position to get position relative to document + // Also: using Math.floor() instead of Math.round() below because in Edge, + // getBoundingClientRect returns partial pixel values (e.g. 162.5px) and Chrome already + // floors the value (e.g. 162px). Keeping behavior consistent across + layout = [ + Math.floor(rect.left) + ("pageXOffset" in window ? window.pageXOffset : doc.scrollLeft), + Math.floor(rect.top) + ("pageYOffset" in window ? window.pageYOffset : doc.scrollTop), + Math.round(rect.width), + Math.round(rect.height) + ]; + } + return layout.join("x"); +} diff --git a/src/instrument/counter.ts b/src/instrument/counter.ts new file mode 100644 index 00000000..3bafd9f3 --- /dev/null +++ b/src/instrument/counter.ts @@ -0,0 +1,46 @@ +import {Method} from "../lib/method"; + +interface ICounter { + [key: number]: ICounterValue; +} + +interface ICounterValue { + start: number; + end: number; + duration: number; + count: number; +} + +let tracker: ICounter = {}; +window["TRACKER"] = tracker; // DEBUG: Remove later +let threshold = 50; + +export function longtasks(method: Method): boolean { + let elapsed = Date.now() - tracker[method].start; + return (elapsed > threshold); +} + +export function start(method: Method): void { + if (!(method in tracker)) { + tracker[method] = { start: 0, end: 0, duration: 0, count: 0 }; + } + tracker[method].start = Date.now(); +} + +export function stop(method: Method): void { + tracker[method].end = Date.now(); + tracker[method].duration += tracker[method]["end"] - tracker[method]["start"]; + tracker[method].count++; +} + +export async function idle(method: Method): Promise { + stop(method); + await wait(); + start(method); +} + +async function wait(): Promise { + return new Promise((resolve: FrameRequestCallback): void => { + requestAnimationFrame(resolve); + }); +} diff --git a/src/lib/hash.ts b/src/lib/hash.ts new file mode 100644 index 00000000..4531f639 --- /dev/null +++ b/src/lib/hash.ts @@ -0,0 +1,12 @@ +export function hash(input: string): number { + let value = 0; + if (input.length === 0) { return value; } + for (let i = 0; i < input.length; i++) { + let char = input.charCodeAt(i); + // tslint:disable-next-line: no-bitwise + value = ((value << 5) - value) + char; + // tslint:disable-next-line: no-bitwise + value = value & value; // 32bit int conversion + } + return value; +} diff --git a/src/lib/method.ts b/src/lib/method.ts new file mode 100644 index 00000000..f2f399b5 --- /dev/null +++ b/src/lib/method.ts @@ -0,0 +1,5 @@ +export enum Method { + Discover, + Mutation, + Layout +} diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts new file mode 100644 index 00000000..37b23404 --- /dev/null +++ b/src/lib/nodetree.ts @@ -0,0 +1,120 @@ +interface IAttributes { + [key: string]: string; +} + +interface INodeData { + attributes?: IAttributes; + layout?: string; + leaf?: boolean; + value?: string; +} + +interface INodeValue { + parent: number; + children: number[]; + active: boolean; + update: boolean; + data: INodeData; +} + +export class NodeTree { + + private static NODE_ID_PROP: string = "__node_index__"; + private static index: number = 0; + + private nodes: Node[]; + private values: INodeValue[]; + + private backupIndex: number; + private backupNodes: Node[]; + private backupValues: Node[]; + + constructor() { + this.nodes = []; + this.values = []; + } + + public id(node: Node): number { + if (node === null) { return null; } + let id = node[NodeTree.NODE_ID_PROP]; + if (!id) { + id = node[NodeTree.NODE_ID_PROP] = NodeTree.index++; + } + return id; + } + + public add(parent: Node, node: Node, data: INodeData): void { + let id = this.id(node); + let parentId = -1; + + if (parent) { + parentId = this.id(parent); + if (this.values[parentId]) { + this.values[parentId].children.push(id); + } + } + + this.nodes[id] = node; + this.values[id] = { + parent: parentId, + children: [], + active: true, + update: true, + data + }; + } + + public update(node: Node, value: INodeValue): void { + let id = this.id(node); + for (let key in value) { + if (key in this.values[id]) { + this.values[id][key] = value[key]; + } + } + } + + public get(node: Node): INodeValue { + let id = this.id(node); + return this.values[id]; + } + + public has(node: Node): boolean { + return this.id(node) in this.nodes; + } + + public remove(node: Node): void { + let id = this.id(node); + this.delete(id); + } + + public keys(): Node[] { + let nodes: Node[] = []; + for (let id in this.nodes) { + if (this.nodes[id]) { + nodes.push(this.nodes[id]); + } + } + return nodes; + } + + public backup(): void { + this.backupNodes = Array.from(this.nodes); + this.backupValues = JSON.parse(JSON.stringify(this.values)); + this.backupIndex = NodeTree.index; + } + + public rollback(): void { + this.nodes = Array.from(this.backupNodes); + this.values = JSON.parse(JSON.stringify(this.backupValues)); + NodeTree.index = this.backupIndex; + } + + private delete(id: number): void { + let children = this.values[id].children; + for (let i = 0; i < children.length; i++) { + this.delete(children[i]); + } + this.values[id].active = false; + this.values[id].update = true; + } +} From 502646a29f0594cfa1f58a95ad539c5ddc0acaec Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 10 Jun 2019 09:39:41 -0700 Subject: [PATCH 003/105] Adding support for serialization --- src/core.ts | 3 ++ src/dom/node.ts | 3 ++ src/dom/serialize.ts | 72 ++++++++++++++++++++++++++++++++++++++++++++ src/lib/nodetree.ts | 14 +++++++-- 4 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 src/dom/serialize.ts diff --git a/src/core.ts b/src/core.ts index e8e0bc35..5c3e3b34 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,4 +1,5 @@ import discover from "./dom/discover"; +import serialize from "./dom/serialize"; /* Initial discovery of DOM */ export function init(): void { @@ -6,5 +7,7 @@ export function init(): void { // DEBUG: Remove later console.log("done discovery!"); console.log(window["TRACKER"][0]["duration"] + "ms in " + window["TRACKER"][0]["count"] + " iterations"); + // DEBUG: Serialize DOM + console.log("Serialized DOM: " + serialize(document.documentElement)); }); } diff --git a/src/dom/node.ts b/src/dom/node.ts index f9d2c6bf..95b8f28f 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -8,6 +8,7 @@ export default function(node: Node): void { case Node.DOCUMENT_TYPE_NODE: let doctype = node as DocumentType; nodes.add(parent, node, { + tag: "*DOC*", attributes: { name: doctype.name, publicId: doctype.publicId, @@ -20,6 +21,7 @@ export default function(node: Node): void { if (parent && nodes.has(parent)) { // nodes.update(parent, {leaf: true}); nodes.add(parent, node, { + tag: "*TXT*", leaf: false, value: node.nodeValue, layout: getTextLayout(node) @@ -35,6 +37,7 @@ export default function(node: Node): void { break; default: nodes.add(parent, node, { + tag: element.tagName, leaf: node.childNodes.length === 0, attributes: getAttributes(element.attributes), layout: getLayout(element) diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts new file mode 100644 index 00000000..cf654a2d --- /dev/null +++ b/src/dom/serialize.ts @@ -0,0 +1,72 @@ +import {nodes} from "../data/state"; +import {INodeData, INodeValue} from "../lib/nodetree"; + +export default function serialize(node: Node): string { + let markup = []; + let children = []; + let value: INodeValue = nodes.get(node); + let data: INodeData = value.data; + let keys = ["tag", "attributes", "layout", "value", "children"]; + for (let key of keys) { + if (data[key] || value[key]) { + switch (key) { + case "tag": + if (data[key] !== "*TXT*") { + markup.push(data[key]); + } + break; + case "attributes": + for (let attr in data[key]) { + if (data[key][attr]) { + markup.push(`"${attr}=${data[key][attr]}"`); + } + } + break; + case "layout": + markup.push(`${data[key]}`); + break; + case "value": + let parent = nodes.node(value.parent); + let parentTag = nodes.get(parent).data.tag; + markup.push(`"${text(parentTag, data[key])}"`); + break; + case "children": + if (value[key].length > 0) { + for (let i = 0; i < value[key].length; i++) { + let childNode = nodes.node(value[key][i]); + children.push(serialize(childNode)); + } + markup.push(`[${children.join(",")}]`); + } + break; + } + } + } + return markup.join(" "); +} + +function text(tag: string, value: string): string { + tag = tag.toLowerCase(); + switch (tag) { + case "style": + case "title": + return value; + default: + let masked = ""; + let activeCode = "x"; + let activeCount = 0; + let currentCode = "y"; + for (let i = 0; i < value.length; i++) { + let code = value.charCodeAt(i); + currentCode = (code === 32 || code === 10 || code === 13) ? "y" : "x"; + if (currentCode !== activeCode) { + masked += activeCount > 0 ? `${activeCode}${activeCount}` : ""; + activeCode = currentCode; + activeCount = 0; + } + activeCount++; + } + masked += `${activeCode}${activeCount}`; + return masked; + } +} diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts index 37b23404..86d1d3e6 100644 --- a/src/lib/nodetree.ts +++ b/src/lib/nodetree.ts @@ -1,15 +1,16 @@ -interface IAttributes { +export interface IAttributes { [key: string]: string; } -interface INodeData { +export interface INodeData { + tag: string; attributes?: IAttributes; layout?: string; leaf?: boolean; value?: string; } -interface INodeValue { +export interface INodeValue { parent: number; children: number[]; active: boolean; @@ -73,6 +74,13 @@ export class NodeTree { } } + public node(id: number): Node { + if (id in this.nodes) { + return this.nodes[id]; + } + return null; + } + public get(node: Node): INodeValue { let id = this.id(node); return this.values[id]; From 66f1c10f6aa1212942a710406a3563e614025fb2 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 12 Jun 2019 10:17:34 -0700 Subject: [PATCH 004/105] Updating layout computation code --- src/dom/node.ts | 2 +- src/dom/serialize.ts | 26 +++++++++++--------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/dom/node.ts b/src/dom/node.ts index 95b8f28f..75e18105 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -94,7 +94,7 @@ function getLayout(element: Element): string { // Ignore } - if (rect) { + if (rect && rect.width > 0 && rect.height > 0) { // getBoundingClientRect returns relative positioning to viewport and therefore needs // addition of window scroll position to get position relative to document // Also: using Math.floor() instead of Math.round() below because in Edge, diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts index cf654a2d..f224ec8a 100644 --- a/src/dom/serialize.ts +++ b/src/dom/serialize.ts @@ -28,7 +28,7 @@ export default function serialize(node: Node): string { case "value": let parent = nodes.node(value.parent); let parentTag = nodes.get(parent).data.tag; - markup.push(`"${text(parentTag, data[key])}"`); + markup.push(`${text(parentTag, data[key])}`); break; case "children": if (value[key].length > 0) { @@ -46,27 +46,23 @@ export default function serialize(node: Node): string { } function text(tag: string, value: string): string { - tag = tag.toLowerCase(); switch (tag) { - case "style": - case "title": + case "STYLE": + case "TITLE": return value; default: let masked = ""; - let activeCode = "x"; - let activeCount = 0; - let currentCode = "y"; + let wasWhiteSpace = false; + let textCount = 0; + let wordCount = 0; for (let i = 0; i < value.length; i++) { let code = value.charCodeAt(i); - currentCode = (code === 32 || code === 10 || code === 13) ? "y" : "x"; - if (currentCode !== activeCode) { - masked += activeCount > 0 ? `${activeCode}${activeCount}` : ""; - activeCode = currentCode; - activeCount = 0; - } - activeCount++; + let isWhiteSpace = (code === 32 || code === 10 || code === 9 || code === 13); + textCount += isWhiteSpace ? 0 : 1; + wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; + wasWhiteSpace = isWhiteSpace; } - masked += `${activeCode}${activeCount}`; + masked += `${textCount}x${wordCount}`; return masked; } } From 1065a9d74db7ee110f629d6296fa2125b4cb712d Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 13 Jun 2019 06:53:30 -0700 Subject: [PATCH 005/105] Flatten dom serialization --- src/dom/serialize.ts | 28 ++++++++++++++++------------ src/lib/nodetree.ts | 2 ++ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts index f224ec8a..56749f9e 100644 --- a/src/dom/serialize.ts +++ b/src/dom/serialize.ts @@ -1,9 +1,13 @@ import {nodes} from "../data/state"; import {INodeData, INodeValue} from "../lib/nodetree"; -export default function serialize(node: Node): string { +export default function(node: Node): string { + let markup = serialize(node); + return JSON.stringify(markup); +} + +function serialize(node: Node): any { let markup = []; - let children = []; let value: INodeValue = nodes.get(node); let data: INodeData = value.data; let keys = ["tag", "attributes", "layout", "value", "children"]; @@ -11,14 +15,14 @@ export default function serialize(node: Node): string { if (data[key] || value[key]) { switch (key) { case "tag": - if (data[key] !== "*TXT*") { - markup.push(data[key]); - } + markup.push(value.id); + markup.push(value.parent); + markup.push(data[key]); break; case "attributes": for (let attr in data[key]) { if (data[key][attr]) { - markup.push(`"${attr}=${data[key][attr]}"`); + markup.push(`${attr}=${data[key][attr]}`); } } break; @@ -34,15 +38,17 @@ export default function serialize(node: Node): string { if (value[key].length > 0) { for (let i = 0; i < value[key].length; i++) { let childNode = nodes.node(value[key][i]); - children.push(serialize(childNode)); + let child = serialize(childNode); + for (let j = 0; j < child.length; j++) { + markup.push(child[j]); + } } - markup.push(`[${children.join(",")}]`); } break; } } } - return markup.join(" "); + return markup; } function text(tag: string, value: string): string { @@ -51,7 +57,6 @@ function text(tag: string, value: string): string { case "TITLE": return value; default: - let masked = ""; let wasWhiteSpace = false; let textCount = 0; let wordCount = 0; @@ -62,7 +67,6 @@ function text(tag: string, value: string): string { wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; wasWhiteSpace = isWhiteSpace; } - masked += `${textCount}x${wordCount}`; - return masked; + return `${textCount}x${wordCount}`; } } diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts index 86d1d3e6..d540f602 100644 --- a/src/lib/nodetree.ts +++ b/src/lib/nodetree.ts @@ -11,6 +11,7 @@ export interface INodeData { } export interface INodeValue { + id: number; parent: number; children: number[]; active: boolean; @@ -57,6 +58,7 @@ export class NodeTree { this.nodes[id] = node; this.values[id] = { + id, parent: parentId, children: [], active: true, From 888b888ed52476245450249c024b3091e7502c5e Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 18 Jun 2019 08:57:22 -0700 Subject: [PATCH 006/105] Adding basic support for mutations --- src/core.ts | 11 +++- src/data/token.ts | 5 ++ src/dom/discover.ts | 3 +- src/dom/mutation.ts | 61 ++++++++++++++++++++++ src/dom/node.ts | 60 +++++++++------------- src/dom/serialize.ts | 120 ++++++++++++++++++++++++++++--------------- src/lib/hash.ts | 9 ++-- src/lib/method.ts | 3 +- src/lib/nodetree.ts | 82 +++++++++++++++++++++++------ 9 files changed, 255 insertions(+), 99 deletions(-) create mode 100644 src/data/token.ts create mode 100644 src/dom/mutation.ts diff --git a/src/core.ts b/src/core.ts index 5c3e3b34..3ce5449f 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,13 +1,22 @@ import discover from "./dom/discover"; +import mutation from "./dom/mutation"; import serialize from "./dom/serialize"; +window["SERIALIZE"] = serialize; + /* Initial discovery of DOM */ export function init(): void { + mutation(); discover().then(() => { // DEBUG: Remove later console.log("done discovery!"); console.log(window["TRACKER"][0]["duration"] + "ms in " + window["TRACKER"][0]["count"] + " iterations"); // DEBUG: Serialize DOM - console.log("Serialized DOM: " + serialize(document.documentElement)); + serialize().then((output: string) => { + console.log("Serialized DOM: " + output); + console.log("Serialized DOM Length: " + output.length); + console.log("done serialization!"); + console.log(window["TRACKER"][3]["duration"] + "ms in " + window["TRACKER"][3]["count"] + " iterations"); + }); }); } diff --git a/src/data/token.ts b/src/data/token.ts new file mode 100644 index 00000000..5b346abf --- /dev/null +++ b/src/data/token.ts @@ -0,0 +1,5 @@ +let tokens: string[] = []; + +export function check(hash: string, length: number): boolean { + return tokens.indexOf(hash) >= 0; +} diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 4e542241..6afcd2e0 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -12,5 +12,6 @@ export default async function(): Promise { processNode(node); node = walker.nextNode(); } - counter.stop(Method.Discover); + console.log("Finished discovering"); + counter.stop(method); } diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts new file mode 100644 index 00000000..6ce930ee --- /dev/null +++ b/src/dom/mutation.ts @@ -0,0 +1,61 @@ +import * as counter from "../instrument/counter"; +import { Method } from "../lib/method"; +import processNode from "./node"; + +let observer: MutationObserver; +window["MUTATIONS"] = []; + +export default function(): void { + console.log("Listening for mutations..."); + if (observer) { + observer.disconnect(); + } + observer = window["MutationObserver"] ? new MutationObserver(handle) : null; + observer.observe(document, { attributes: true, childList: true, characterData: true, subtree: true }); +} + +function handle(mutations: MutationRecord[]): void { + let method = Method.Mutation; + counter.start(method); + let length = mutations.length; + for (let i = 0; i < length; i++) { + process(mutations[i]); + } + counter.stop(method); +} + +async function process(mutation: MutationRecord): Promise { + let method = Method.Mutation; + console.log("Received mutation: " + mutation.type + " | " + mutation.target); + window["MUTATIONS"].push(mutation); + + let target = mutation.target; + switch (mutation.type) { + case "attributes": + case "characterData": + if (counter.longtasks(method)) { await counter.idle(method); } + processNode(target); + break; + case "childList": + // Process additions + let addedLength = mutation.addedNodes.length; + for (let j = 0; j < addedLength; j++) { + let walker = document.createTreeWalker(mutation.addedNodes[j], NodeFilter.SHOW_ALL, null, false); + let node = walker.currentNode; + while (node) { + if (counter.longtasks(method)) { await counter.idle(method); } + processNode(node); + node = walker.nextNode(); + } + } + // Process removes + let removedLength = mutation.removedNodes.length; + for (let j = 0; j < removedLength; j++) { + if (counter.longtasks(method)) { await counter.idle(method); } + processNode(mutation.removedNodes[j]); + } + break; + default: + break; + } +} diff --git a/src/dom/node.ts b/src/dom/node.ts index 75e18105..243ec2af 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -1,47 +1,37 @@ import {nodes} from "../data/state"; -let ignoreAttributes = ["title", "alt"]; +let ignoreAttributes = ["title", "alt", "onload", "onfocus"]; export default function(node: Node): void { - let parent = node.parentElement; + let call = nodes.has(node) ? "update" : "add"; switch (node.nodeType) { case Node.DOCUMENT_TYPE_NODE: let doctype = node as DocumentType; - nodes.add(parent, node, { - tag: "*DOC*", - attributes: { - name: doctype.name, - publicId: doctype.publicId, - systemId: doctype.systemId - } - }); + let docAttributes = { name: doctype.name, publicId: doctype.publicId, systemId: doctype.systemId }; + let docData = { tag: "*D", attributes: docAttributes }; + nodes[call](node, docData); break; case Node.TEXT_NODE: // Account for this text node only if we are tracking the parent node + // We do not wish to track text nodes for ignored parent nodes, like script tags + let parent = node.parentElement; if (parent && nodes.has(parent)) { - // nodes.update(parent, {leaf: true}); - nodes.add(parent, node, { - tag: "*TXT*", - leaf: false, - value: node.nodeValue, - layout: getTextLayout(node) - }); + let textData = { tag: "*T", value: node.nodeValue }; + textData["layout"] = getTextLayout(node); + nodes[call](node, textData); } break; case Node.ELEMENT_NODE: - let element = ( node as HTMLElement); + let element = (node as HTMLElement); switch (element.tagName) { case "SCRIPT": case "NOSCRIPT": case "META": break; default: - nodes.add(parent, node, { - tag: element.tagName, - leaf: node.childNodes.length === 0, - attributes: getAttributes(element.attributes), - layout: getLayout(element) - }); + let data = { tag: element.tagName, attributes: getAttributes(element.attributes) }; + data["layout"] = getLayout(element); + nodes[call](node, data); break; } break; @@ -63,25 +53,25 @@ function getAttributes(attributes: NamedNodeMap): {[key: string]: string} { return output; } -function getTextLayout(textNode: Node): string { - let layouts: string[] = []; +function getTextLayout(textNode: Node): number[] { + return []; + let layout: number[] = []; let range = document.createRange(); range.selectNodeContents(textNode); let rects = range.getClientRects(); let doc = document.documentElement; for (let i = 0; i < rects.length; i++) { let rect = rects[i]; - layouts.push([ - Math.floor(rect.left) + ("pageXOffset" in window ? window.pageXOffset : doc.scrollLeft), - Math.floor(rect.top) + ("pageYOffset" in window ? window.pageYOffset : doc.scrollTop), - Math.round(rect.width), - Math.round(rect.height) - ].join("x")); + layout.push(Math.floor(rect.left) + ("pageXOffset" in window ? window.pageXOffset : doc.scrollLeft)); + layout.push(Math.floor(rect.top) + ("pageYOffset" in window ? window.pageYOffset : doc.scrollTop)); + layout.push(Math.round(rect.width)); + layout.push(Math.round(rect.height)); } - return layouts.join("."); + return layout; } -function getLayout(element: Element): string { +function getLayout(element: Element): number[] { + return []; // In IE, calling getBoundingClientRect on a node that is disconnected // from a DOM tree, sometimes results in a 'Unspecified Error' // Wrapping this in try/catch is faster than checking whether element is connected to DOM @@ -107,5 +97,5 @@ function getLayout(element: Element): string { Math.round(rect.height) ]; } - return layout.join("x"); + return layout; } diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts index 56749f9e..03c8f599 100644 --- a/src/dom/serialize.ts +++ b/src/dom/serialize.ts @@ -1,54 +1,74 @@ import {nodes} from "../data/state"; +import {check} from "../data/token"; +import * as counter from "../instrument/counter"; +import hash from "../lib/hash"; +import { Method } from "../lib/method"; import {INodeData, INodeValue} from "../lib/nodetree"; -export default function(node: Node): string { - let markup = serialize(node); - return JSON.stringify(markup); -} +window["HASH"] = hash; +let reference: number = 0; -function serialize(node: Node): any { +export default async function(): Promise { + let method = Method.Serialize; + counter.start(method); let markup = []; - let value: INodeValue = nodes.get(node); - let data: INodeData = value.data; - let keys = ["tag", "attributes", "layout", "value", "children"]; - for (let key of keys) { - if (data[key] || value[key]) { - switch (key) { - case "tag": - markup.push(value.id); - markup.push(value.parent); - markup.push(data[key]); - break; - case "attributes": - for (let attr in data[key]) { - if (data[key][attr]) { - markup.push(`${attr}=${data[key][attr]}`); - } - } - break; - case "layout": - markup.push(`${data[key]}`); - break; - case "value": - let parent = nodes.node(value.parent); - let parentTag = nodes.get(parent).data.tag; - markup.push(`${text(parentTag, data[key])}`); - break; - case "children": - if (value[key].length > 0) { - for (let i = 0; i < value[key].length; i++) { - let childNode = nodes.node(value[key][i]); - let child = serialize(childNode); - for (let j = 0; j < child.length; j++) { - markup.push(child[j]); + let values: INodeValue[] = nodes.getValues(); + reference = 0; + for (let value of values) { + if (counter.longtasks(method)) { await counter.idle(method); } + let metadata = []; + let data: INodeData = value.data; + let keys = ["tag", "attributes", "layout", "value"]; + for (let key of keys) { + if (data[key]) { + switch (key) { + case "tag": + markup.push(number(value.parent)); + markup.push(number(value.previous)); + markup.push(number(value.id)); + metadata.push(data[key]); + break; + case "attributes": + for (let attr in data[key]) { + if (data[key][attr]) { + metadata.push(`${attr}=${data[key][attr]}`); } } - } - break; + break; + case "layout": + if (data[key].length > 0) { + markup.push(layout(data[key])); + } + break; + case "value": + let parent = nodes.node(value.parent); + let parentTag = nodes.get(parent).data.tag; + metadata.push(text(parentTag, data[key])); + break; + } } } + + // Add metadata + metadata = meta(metadata); + for (let token of metadata) { + let index: number = typeof token === "string" ? markup.indexOf(token) : -1; + markup.push(index >= 0 ? [index] : token); + } + + // Mark this node as processed, so we don't serialize it again + value["update"] = false; } - return markup; + let json = JSON.stringify(markup); + counter.stop(method); + return json; +} + +function meta(metadata: string[]): string[] | string[][] { + let value = JSON.stringify(metadata); + let length = value.length; + let hashed = hash(value); + return check(hashed, length) ? [[hashed]] : metadata; } function text(tag: string, value: string): string { @@ -57,6 +77,7 @@ function text(tag: string, value: string): string { case "TITLE": return value; default: + return value; let wasWhiteSpace = false; let textCount = 0; let wordCount = 0; @@ -70,3 +91,20 @@ function text(tag: string, value: string): string { return `${textCount}x${wordCount}`; } } + +function layout(l: number[]): string { + let output = []; + for (let i = 0; i < l.length; i = i + 4) { + output.push([l[i + 0].toString(36), l[i + 1].toString(36), l[i + 2].toString(36), l[i + 3].toString(36)].join(".")); + } + return output.join("|"); +} + +function number(id: number): number { + let output = id; + if (id > 0) { + output = id - reference; + reference = id; + } + return output; +} diff --git a/src/lib/hash.ts b/src/lib/hash.ts index 4531f639..43af6bd5 100644 --- a/src/lib/hash.ts +++ b/src/lib/hash.ts @@ -1,12 +1,11 @@ -export function hash(input: string): number { +// tslint:disable: no-bitwise +export default function(input: string): string { let value = 0; - if (input.length === 0) { return value; } for (let i = 0; i < input.length; i++) { let char = input.charCodeAt(i); - // tslint:disable-next-line: no-bitwise value = ((value << 5) - value) + char; - // tslint:disable-next-line: no-bitwise value = value & value; // 32bit int conversion } - return value; + value = value & 0xfffffff; + return value.toString(36); } diff --git a/src/lib/method.ts b/src/lib/method.ts index f2f399b5..8ae0c9c3 100644 --- a/src/lib/method.ts +++ b/src/lib/method.ts @@ -1,5 +1,6 @@ export enum Method { Discover, Mutation, - Layout + Layout, + Serialize } diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts index d540f602..e69eccd1 100644 --- a/src/lib/nodetree.ts +++ b/src/lib/nodetree.ts @@ -5,14 +5,14 @@ export interface IAttributes { export interface INodeData { tag: string; attributes?: IAttributes; - layout?: string; - leaf?: boolean; + layout?: number[]; value?: string; } export interface INodeValue { id: number; parent: number; + previous: number; children: number[]; active: boolean; update: boolean; @@ -22,7 +22,7 @@ export interface INodeValue { export class NodeTree { private static NODE_ID_PROP: string = "__node_index__"; - private static index: number = 0; + private static index: number = 1; private nodes: Node[]; private values: INodeValue[]; @@ -45,21 +45,20 @@ export class NodeTree { return id; } - public add(parent: Node, node: Node, data: INodeData): void { + public add(node: Node, data: INodeData): void { let id = this.id(node); - let parentId = -1; + let parentId = node.parentElement ? this.id(node.parentElement) : 0; + let previousId = node.previousSibling ? this.id(node.previousSibling) : 0; - if (parent) { - parentId = this.id(parent); - if (this.values[parentId]) { - this.values[parentId].children.push(id); - } + if (parentId >= 0 && this.values[parentId]) { + this.values[parentId].children.push(id); } this.nodes[id] = node; this.values[id] = { id, parent: parentId, + previous: previousId, children: [], active: true, update: true, @@ -67,12 +66,55 @@ export class NodeTree { }; } - public update(node: Node, value: INodeValue): void { + public update(node: Node, data: INodeData): void { let id = this.id(node); - for (let key in value) { - if (key in this.values[id]) { - this.values[id][key] = value[key]; + console.log("Updating node: " + id); + let parentId = node.parentElement ? this.id(node.parentElement) : 0; + let previousId = node.previousSibling ? this.id(node.previousSibling) : 0; + + if (id in this.values) { + let value = this.values[id]; + console.log("Previous value: " + JSON.stringify(value)); + + // Handle case where internal ordering may have changed + if (value["previous"] !== previousId) { + let oldPreviousId = value["previous"]; + value["previous"] = previousId; + console.log("Old previous id: " + oldPreviousId + " | " + previousId); + } + + // Handle case where parent might have been updated + if (value["parent"] !== parentId) { + let oldParentId = value["parent"]; + value["parent"] = parentId; + console.log("Old parent id: " + oldParentId + " | " + parentId); + // Move this node to the right location under new parent + if (parentId >= 0) { + if (previousId >= 0) { + this.values[parentId].children.splice(previousId + 1, 0 , id); + } else { + this.values[parentId].children.push(id); + } + } else { + // Mark this element as deleted if the parent has been updated to null + value["active"] = false; + } + + // Remove reference to this node from the old parent + let index = this.values[oldParentId].children.indexOf(id); + if (index >= 0) { + this.values[oldParentId].children.splice(index, 1); + } + } + + // Update data + for (let key in data) { + if (key in value["data"]) { + value["data"][key] = data[key]; + } } + + value["update"] = true; } } @@ -97,7 +139,7 @@ export class NodeTree { this.delete(id); } - public keys(): Node[] { + public getNodes(): Node[] { let nodes: Node[] = []; for (let id in this.nodes) { if (this.nodes[id]) { @@ -107,6 +149,16 @@ export class NodeTree { return nodes; } + public getValues(): INodeValue[] { + let values = []; + for (let id in this.values) { + if (this.values[id] && this.values[id]["update"] === true) { + values.push(this.values[id]); + } + } + return values; + } + public backup(): void { this.backupNodes = Array.from(this.nodes); this.backupValues = JSON.parse(JSON.stringify(this.values)); From d537bc655a0fd2f1eb11e16776f8b4543f3f1b89 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 26 Jun 2019 07:08:57 -0700 Subject: [PATCH 007/105] Adding basic support for deserialization --- src/core.ts | 5 +++++ src/data/token.ts | 12 +++++++++--- src/dom/deserialize.ts | 37 +++++++++++++++++++++++++++++++++++++ src/dom/serialize.ts | 5 ++--- src/lib/nodetree.ts | 4 ++-- 5 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 src/dom/deserialize.ts diff --git a/src/core.ts b/src/core.ts index 3ce5449f..377ea47e 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,3 +1,4 @@ +import deserialize from "./dom/deserialize"; import discover from "./dom/discover"; import mutation from "./dom/mutation"; import serialize from "./dom/serialize"; @@ -17,6 +18,10 @@ export function init(): void { console.log("Serialized DOM Length: " + output.length); console.log("done serialization!"); console.log(window["TRACKER"][3]["duration"] + "ms in " + window["TRACKER"][3]["count"] + " iterations"); + console.log("===================="); + let deserialized = deserialize(output); + console.log("Deserialized DOM: " + deserialized); + console.log("Deserialized DOM Length: " + deserialized.length); }); }); } diff --git a/src/data/token.ts b/src/data/token.ts index 5b346abf..8f1bfbd6 100644 --- a/src/data/token.ts +++ b/src/data/token.ts @@ -1,5 +1,11 @@ -let tokens: string[] = []; +let tokens = {}; -export function check(hash: string, length: number): boolean { - return tokens.indexOf(hash) >= 0; +export function check(hash: string, metadata: string[]): boolean { + let output = hash in tokens; + tokens[hash] = metadata; + return output; +} + +export function resolve(hash: string): string[] { + return hash in tokens ? tokens[hash] : []; } diff --git a/src/dom/deserialize.ts b/src/dom/deserialize.ts new file mode 100644 index 00000000..12506b24 --- /dev/null +++ b/src/dom/deserialize.ts @@ -0,0 +1,37 @@ +import { resolve } from "@src/data/token"; + +export default function(payload: string): string { + let json = JSON.parse(payload); + let tokens = []; + let number = 0; + for (let token of json) { + let type = typeof(token); + switch (type) { + case "number": + number += token; + token = token === 0 ? token : number; + tokens.push(token); + break; + case "string": + tokens.push(token); + break; + case "object": + let subtoken = token[0]; + let subtype = typeof(subtoken); + switch (subtype) { + case "string": + let keys = resolve(token); + for (let key of keys) { + tokens.push(key); + } + break; + case "number": + token = tokens.length > subtoken ? tokens[subtoken] : null; + tokens.push(token); + break; + } + } + } + + return JSON.stringify(tokens); +} diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts index 03c8f599..ae7a6c16 100644 --- a/src/dom/serialize.ts +++ b/src/dom/serialize.ts @@ -53,7 +53,7 @@ export default async function(): Promise { metadata = meta(metadata); for (let token of metadata) { let index: number = typeof token === "string" ? markup.indexOf(token) : -1; - markup.push(index >= 0 ? [index] : token); + markup.push(index >= 0 && token.length > index.toString().length ? [index] : token); } // Mark this node as processed, so we don't serialize it again @@ -66,9 +66,8 @@ export default async function(): Promise { function meta(metadata: string[]): string[] | string[][] { let value = JSON.stringify(metadata); - let length = value.length; let hashed = hash(value); - return check(hashed, length) ? [[hashed]] : metadata; + return check(hashed, metadata) && hashed.length < value.length ? [[hashed]] : metadata; } function text(tag: string, value: string): string { diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts index e69eccd1..8170743c 100644 --- a/src/lib/nodetree.ts +++ b/src/lib/nodetree.ts @@ -14,9 +14,9 @@ export interface INodeValue { parent: number; previous: number; children: number[]; - active: boolean; - update: boolean; data: INodeData; + active?: boolean; + update?: boolean; } export class NodeTree { From 33a56bc2e7b96a3575b8eee496b392d3d9a83dde Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 5 Jul 2019 10:02:09 -0700 Subject: [PATCH 008/105] Adding basic support for mutations --- src/core.ts | 1 + src/dom/deserialize.ts | 127 ++++++++++++++++++++++++++++++++++++++--- src/dom/node.ts | 2 - src/dom/serialize.ts | 24 +++++--- src/lib/nodetree.ts | 30 +++++----- 5 files changed, 151 insertions(+), 33 deletions(-) diff --git a/src/core.ts b/src/core.ts index 377ea47e..895806a2 100644 --- a/src/core.ts +++ b/src/core.ts @@ -4,6 +4,7 @@ import mutation from "./dom/mutation"; import serialize from "./dom/serialize"; window["SERIALIZE"] = serialize; +window["DESERIALIZE"] = deserialize; /* Initial discovery of DOM */ export function init(): void { diff --git a/src/dom/deserialize.ts b/src/dom/deserialize.ts index 12506b24..1db63552 100644 --- a/src/dom/deserialize.ts +++ b/src/dom/deserialize.ts @@ -1,19 +1,35 @@ import { resolve } from "@src/data/token"; +let nodes = {}; +let placeholder = document.createElement("iframe"); +// debug: visualize it on page +placeholder.height = "350"; +document.body.appendChild(placeholder); + export default function(payload: string): string { - let json = JSON.parse(payload); - let tokens = []; + let tokens = JSON.parse(payload); let number = 0; - for (let token of json) { + let lastType = null; + let node = []; + let intermediate = []; + let tagIndex = 0; + for (let token of tokens) { let type = typeof(token); switch (type) { case "number": + if (type !== lastType && lastType !== null) { + process(node, tagIndex); + intermediate.push(...node); + node = []; + tagIndex = 0; + } number += token; token = token === 0 ? token : number; - tokens.push(token); + node.push(token); + tagIndex++; break; case "string": - tokens.push(token); + node.push(token); break; case "object": let subtoken = token[0]; @@ -22,16 +38,111 @@ export default function(payload: string): string { case "string": let keys = resolve(token); for (let key of keys) { - tokens.push(key); + node.push(key); } break; case "number": token = tokens.length > subtoken ? tokens[subtoken] : null; - tokens.push(token); + node.push(token); break; } } + lastType = type; + } + + let markup = placeholder.contentDocument.documentElement.outerHTML; + + return JSON.stringify(intermediate) + "\r\n" + markup; +} + +function process(node: any[] | number[], tagIndex: number): void { + let id = node[0]; + let parent = tagIndex > 1 ? element(node[1]) : null; + let next = tagIndex > 2 ? element(node[2]) : null; + let layouts = []; + let tag = node[tagIndex]; + let doc = placeholder.contentDocument; + let content = null; + let attributes = {}; + + for (let i = tagIndex + 1; i < node.length; i++) { + let token = node[i] as string; + let keyIndex = token.indexOf("="); + let parts = token.split("*"); + if (tag !== "*T" && keyIndex > 0) { + attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); + } else if (parts.length === 4) { + let layout = []; + for (let part of parts) { + layout.push(parseInt(part, 36)); + } + layouts.push(layout); + } else if (tag === "*T" && parts.length === 2) { + let textCount = parseInt(parts[0], 36); + let wordCount = parseInt(parts[1], 36); + content = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); + } else if (tag === "*T") { + content = token; + } + } + + switch (tag) { + case "*D": + if (typeof XMLSerializer !== "undefined" && false) { + doc.open(); + doc.write(new XMLSerializer().serializeToString( + doc.implementation.createDocumentType( + attributes["name"], + attributes["publicId"], + attributes["systemId"] + ) + )); + doc.close(); + } + break; + case "*T": + let textElement = element(id); + textElement = textElement ? textElement : doc.createTextNode(null); + textElement.nodeValue = content; + insert(id, parent, textElement, next); + break; + case "HTML": + let newDoc = doc.implementation.createHTMLDocument(""); + let html = newDoc.documentElement; + let pointer = doc.importNode(html, true); + doc.replaceChild(pointer, doc.documentElement); + if (doc.head) { doc.head.parentNode.removeChild(doc.head); } + if (doc.body) { doc.body.parentNode.removeChild(doc.body); } + nodes[id] = doc.documentElement; + break; + default: + let domElement = element(id); + domElement = domElement ? domElement : doc.createElement(tag); + attributes["data-id"] = id; + setAttributes(domElement as HTMLElement, attributes); + insert(id, parent, domElement, next); + break; } +} + +function element(nodeId: number): Node { + return nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; +} + +function insert(id: number, parent: Node, node: Node, next: Node): void { + parent.insertBefore(node, next); + nodes[id] = node; +} - return JSON.stringify(tokens); +function setAttributes(node: HTMLElement, attributes: object): void { + for (let attribute in node.attributes) { + if (node.hasAttribute(attribute)) { + node.removeAttribute(attribute); + } + } + for (let attribute in attributes) { + if (attributes[attribute]) { + node.setAttribute(attribute, attributes[attribute]); + } + } } diff --git a/src/dom/node.ts b/src/dom/node.ts index 243ec2af..685ca012 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -54,7 +54,6 @@ function getAttributes(attributes: NamedNodeMap): {[key: string]: string} { } function getTextLayout(textNode: Node): number[] { - return []; let layout: number[] = []; let range = document.createRange(); range.selectNodeContents(textNode); @@ -71,7 +70,6 @@ function getTextLayout(textNode: Node): number[] { } function getLayout(element: Element): number[] { - return []; // In IE, calling getBoundingClientRect on a node that is disconnected // from a DOM tree, sometimes results in a 'Unspecified Error' // Wrapping this in try/catch is faster than checking whether element is connected to DOM diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts index ae7a6c16..ecb00d99 100644 --- a/src/dom/serialize.ts +++ b/src/dom/serialize.ts @@ -17,15 +17,16 @@ export default async function(): Promise { for (let value of values) { if (counter.longtasks(method)) { await counter.idle(method); } let metadata = []; + let layouts = []; let data: INodeData = value.data; - let keys = ["tag", "attributes", "layout", "value"]; + let keys = ["tag", "layout", "attributes", "value"]; for (let key of keys) { if (data[key]) { switch (key) { case "tag": - markup.push(number(value.parent)); - markup.push(number(value.previous)); markup.push(number(value.id)); + if (value.parent) { markup.push(number(value.parent)); } + if (value.next) { markup.push(number(value.next)); } metadata.push(data[key]); break; case "attributes": @@ -37,7 +38,10 @@ export default async function(): Promise { break; case "layout": if (data[key].length > 0) { - markup.push(layout(data[key])); + let boxes = layout(data[key]); + for (let box of boxes) { + layouts.push(box); + } } break; case "value": @@ -55,6 +59,10 @@ export default async function(): Promise { let index: number = typeof token === "string" ? markup.indexOf(token) : -1; markup.push(index >= 0 && token.length > index.toString().length ? [index] : token); } + // Add layout boxes + for (let entry of layouts) { + markup.push(entry); + } // Mark this node as processed, so we don't serialize it again value["update"] = false; @@ -87,16 +95,16 @@ function text(tag: string, value: string): string { wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; wasWhiteSpace = isWhiteSpace; } - return `${textCount}x${wordCount}`; + return `${textCount.toString(36)}*${wordCount.toString(36)}`; } } -function layout(l: number[]): string { +function layout(l: number[]): string[] { let output = []; for (let i = 0; i < l.length; i = i + 4) { - output.push([l[i + 0].toString(36), l[i + 1].toString(36), l[i + 2].toString(36), l[i + 3].toString(36)].join(".")); + output.push([l[i + 0].toString(36), l[i + 1].toString(36), l[i + 2].toString(36), l[i + 3].toString(36)].join("*")); } - return output.join("|"); + return output; } function number(id: number): number { diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts index 8170743c..e4ec4ed6 100644 --- a/src/lib/nodetree.ts +++ b/src/lib/nodetree.ts @@ -12,7 +12,7 @@ export interface INodeData { export interface INodeValue { id: number; parent: number; - previous: number; + next: number; children: number[]; data: INodeData; active?: boolean; @@ -36,19 +36,19 @@ export class NodeTree { this.values = []; } - public id(node: Node): number { + public id(node: Node, autogen: boolean = true): number { if (node === null) { return null; } let id = node[NodeTree.NODE_ID_PROP]; - if (!id) { + if (!id && autogen) { id = node[NodeTree.NODE_ID_PROP] = NodeTree.index++; } - return id; + return id ? id : null; } public add(node: Node, data: INodeData): void { let id = this.id(node); - let parentId = node.parentElement ? this.id(node.parentElement) : 0; - let previousId = node.previousSibling ? this.id(node.previousSibling) : 0; + let parentId = node.parentElement ? this.id(node.parentElement) : null; + let nextId = node.nextSibling ? this.id(node.nextSibling, false) : null; if (parentId >= 0 && this.values[parentId]) { this.values[parentId].children.push(id); @@ -58,7 +58,7 @@ export class NodeTree { this.values[id] = { id, parent: parentId, - previous: previousId, + next: nextId, children: [], active: true, update: true, @@ -69,18 +69,18 @@ export class NodeTree { public update(node: Node, data: INodeData): void { let id = this.id(node); console.log("Updating node: " + id); - let parentId = node.parentElement ? this.id(node.parentElement) : 0; - let previousId = node.previousSibling ? this.id(node.previousSibling) : 0; + let parentId = node.parentElement ? this.id(node.parentElement) : null; + let nextId = node.nextSibling ? this.id(node.nextSibling) : null; if (id in this.values) { let value = this.values[id]; console.log("Previous value: " + JSON.stringify(value)); // Handle case where internal ordering may have changed - if (value["previous"] !== previousId) { - let oldPreviousId = value["previous"]; - value["previous"] = previousId; - console.log("Old previous id: " + oldPreviousId + " | " + previousId); + if (value["next"] !== nextId) { + let oldNextId = value["next"]; + value["next"] = nextId; + console.log("Old next id: " + oldNextId + " | " + nextId); } // Handle case where parent might have been updated @@ -90,8 +90,8 @@ export class NodeTree { console.log("Old parent id: " + oldParentId + " | " + parentId); // Move this node to the right location under new parent if (parentId >= 0) { - if (previousId >= 0) { - this.values[parentId].children.splice(previousId + 1, 0 , id); + if (nextId >= 0) { + this.values[parentId].children.splice(nextId + 1, 0 , id); } else { this.values[parentId].children.push(id); } From 58248dfd4f206a5ac38e4351ea52dce13fd73b59 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 8 Jul 2019 09:18:18 -0700 Subject: [PATCH 009/105] Wiring up metrics code --- src/core.ts | 4 ++-- src/{dom => data}/deserialize.ts | 0 src/data/metric.ts | 3 +++ src/{dom => data}/serialize.ts | 6 +++--- src/dom/discover.ts | 2 +- src/dom/mutation.ts | 2 +- src/instrument/counter.ts | 2 +- src/lib/enums.ts | 27 +++++++++++++++++++++++++++ src/lib/method.ts | 6 ------ 9 files changed, 38 insertions(+), 14 deletions(-) rename src/{dom => data}/deserialize.ts (100%) create mode 100644 src/data/metric.ts rename src/{dom => data}/serialize.ts (97%) create mode 100644 src/lib/enums.ts delete mode 100644 src/lib/method.ts diff --git a/src/core.ts b/src/core.ts index 895806a2..636f98b8 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,7 +1,7 @@ -import deserialize from "./dom/deserialize"; +import deserialize from "./data/deserialize"; +import serialize from "./data/serialize"; import discover from "./dom/discover"; import mutation from "./dom/mutation"; -import serialize from "./dom/serialize"; window["SERIALIZE"] = serialize; window["DESERIALIZE"] = deserialize; diff --git a/src/dom/deserialize.ts b/src/data/deserialize.ts similarity index 100% rename from src/dom/deserialize.ts rename to src/data/deserialize.ts diff --git a/src/data/metric.ts b/src/data/metric.ts new file mode 100644 index 00000000..ed266eae --- /dev/null +++ b/src/data/metric.ts @@ -0,0 +1,3 @@ +export function set(): void { + return null; +} diff --git a/src/dom/serialize.ts b/src/data/serialize.ts similarity index 97% rename from src/dom/serialize.ts rename to src/data/serialize.ts index ecb00d99..1e5def44 100644 --- a/src/dom/serialize.ts +++ b/src/data/serialize.ts @@ -1,9 +1,9 @@ -import {nodes} from "../data/state"; -import {check} from "../data/token"; import * as counter from "../instrument/counter"; +import { Method } from "../lib/enums"; import hash from "../lib/hash"; -import { Method } from "../lib/method"; import {INodeData, INodeValue} from "../lib/nodetree"; +import {nodes} from "./state"; +import {check} from "./token"; window["HASH"] = hash; let reference: number = 0; diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 6afcd2e0..27da30eb 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,5 +1,5 @@ import * as counter from "../instrument/counter"; -import { Method } from "../lib/method"; +import { Method } from "../lib/enums"; import processNode from "./node"; export default async function(): Promise { diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 6ce930ee..11cfdcc3 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,5 +1,5 @@ import * as counter from "../instrument/counter"; -import { Method } from "../lib/method"; +import { Method } from "../lib/enums"; import processNode from "./node"; let observer: MutationObserver; diff --git a/src/instrument/counter.ts b/src/instrument/counter.ts index 3bafd9f3..f5a68259 100644 --- a/src/instrument/counter.ts +++ b/src/instrument/counter.ts @@ -1,4 +1,4 @@ -import {Method} from "../lib/method"; +import {Method} from "../lib/enums"; interface ICounter { [key: number]: ICounterValue; diff --git a/src/lib/enums.ts b/src/lib/enums.ts new file mode 100644 index 00000000..d43688fb --- /dev/null +++ b/src/lib/enums.ts @@ -0,0 +1,27 @@ +export enum Method { + Discover, + Mutation, + Layout, + Serialize +} + +export enum Metric { + /* Core */ + Wireup, + Bytes, + /* DOM */ + DiscoverCount, + MutationCount, + /* Pointer */ + PointerDistance, + SwipeCount, + /* Viewport */ + ViewportX, + ViewportY, + ViewportWidth, + ViewportHeight, + DocumentWidth, + DocumentHeight, + ActiveTime, + TotalTime +} diff --git a/src/lib/method.ts b/src/lib/method.ts deleted file mode 100644 index 8ae0c9c3..00000000 --- a/src/lib/method.ts +++ /dev/null @@ -1,6 +0,0 @@ -export enum Method { - Discover, - Mutation, - Layout, - Serialize -} From cb57a1e139216d3d389fba404235ca721e943069 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 10 Jul 2019 08:53:59 -0700 Subject: [PATCH 010/105] Reoganizing metrics and serialization code --- src/core.ts | 2 +- src/data/metric.ts | 3 - .../{serialize.ts => serialization/dom.ts} | 27 ++++---- src/data/serialization/metrics.ts | 0 src/dom/discover.ts | 12 ++-- src/dom/mutation.ts | 8 +-- src/instrument/counter.ts | 46 ------------- src/lib/enums.ts | 27 -------- src/lib/nodetree.ts | 5 +- src/metrics/counter.ts | 38 +++++++++++ src/metrics/enums.ts | 31 +++++++++ src/metrics/histogram.ts | 49 ++++++++++++++ src/metrics/mark.ts | 32 +++++++++ src/metrics/timer.ts | 67 +++++++++++++++++++ 14 files changed, 244 insertions(+), 103 deletions(-) delete mode 100644 src/data/metric.ts rename src/data/{serialize.ts => serialization/dom.ts} (86%) create mode 100644 src/data/serialization/metrics.ts delete mode 100644 src/instrument/counter.ts delete mode 100644 src/lib/enums.ts create mode 100644 src/metrics/counter.ts create mode 100644 src/metrics/enums.ts create mode 100644 src/metrics/histogram.ts create mode 100644 src/metrics/mark.ts create mode 100644 src/metrics/timer.ts diff --git a/src/core.ts b/src/core.ts index 636f98b8..df884547 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,5 +1,5 @@ import deserialize from "./data/deserialize"; -import serialize from "./data/serialize"; +import serialize from "./data/serialization/dom"; import discover from "./dom/discover"; import mutation from "./dom/mutation"; diff --git a/src/data/metric.ts b/src/data/metric.ts deleted file mode 100644 index ed266eae..00000000 --- a/src/data/metric.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function set(): void { - return null; -} diff --git a/src/data/serialize.ts b/src/data/serialization/dom.ts similarity index 86% rename from src/data/serialize.ts rename to src/data/serialization/dom.ts index 1e5def44..9c2897b0 100644 --- a/src/data/serialize.ts +++ b/src/data/serialization/dom.ts @@ -1,21 +1,22 @@ -import * as counter from "../instrument/counter"; -import { Method } from "../lib/enums"; -import hash from "../lib/hash"; -import {INodeData, INodeValue} from "../lib/nodetree"; -import {nodes} from "./state"; -import {check} from "./token"; +import hash from "../../lib/hash"; +import {INodeData} from "../../lib/nodetree"; +import * as counter from "../../metrics/counter"; +import { Counter, Timer } from "../../metrics/enums"; +import * as timer from "../../metrics/timer"; +import {nodes} from "../state"; +import {check} from "../token"; window["HASH"] = hash; let reference: number = 0; export default async function(): Promise { - let method = Method.Serialize; - counter.start(method); + let method = Timer.Serialize; + timer.start(method); let markup = []; - let values: INodeValue[] = nodes.getValues(); + let values = nodes.summarize(); reference = 0; for (let value of values) { - if (counter.longtasks(method)) { await counter.idle(method); } + if (timer.longtasks(method)) { await timer.idle(method); } let metadata = []; let layouts = []; let data: INodeData = value.data; @@ -24,6 +25,7 @@ export default async function(): Promise { if (data[key]) { switch (key) { case "tag": + counter.increment(Counter.Nodes); markup.push(number(value.id)); if (value.parent) { markup.push(number(value.parent)); } if (value.next) { markup.push(number(value.next)); } @@ -63,12 +65,9 @@ export default async function(): Promise { for (let entry of layouts) { markup.push(entry); } - - // Mark this node as processed, so we don't serialize it again - value["update"] = false; } let json = JSON.stringify(markup); - counter.stop(method); + timer.stop(method); return json; } diff --git a/src/data/serialization/metrics.ts b/src/data/serialization/metrics.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 27da30eb..a5e32c4e 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,17 +1,17 @@ -import * as counter from "../instrument/counter"; -import { Method } from "../lib/enums"; +import { Timer } from "../metrics/enums"; +import * as timer from "../metrics/timer"; import processNode from "./node"; export default async function(): Promise { - let method = Method.Discover; - counter.start(method); + let method = Timer.Discover; + timer.start(method); let walker = document.createTreeWalker(document, NodeFilter.SHOW_ALL, null, false); let node = walker.nextNode(); while (node) { - if (counter.longtasks(method)) { await counter.idle(method); } + if (timer.longtasks(method)) { await timer.idle(method); } processNode(node); node = walker.nextNode(); } console.log("Finished discovering"); - counter.stop(method); + timer.stop(method); } diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 11cfdcc3..e6bcec42 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,5 +1,5 @@ -import * as counter from "../instrument/counter"; -import { Method } from "../lib/enums"; +import { Timer } from "../metrics/enums"; +import * as counter from "../metrics/timer"; import processNode from "./node"; let observer: MutationObserver; @@ -15,7 +15,7 @@ export default function(): void { } function handle(mutations: MutationRecord[]): void { - let method = Method.Mutation; + let method = Timer.Mutation; counter.start(method); let length = mutations.length; for (let i = 0; i < length; i++) { @@ -25,7 +25,7 @@ function handle(mutations: MutationRecord[]): void { } async function process(mutation: MutationRecord): Promise { - let method = Method.Mutation; + let method = Timer.Mutation; console.log("Received mutation: " + mutation.type + " | " + mutation.target); window["MUTATIONS"].push(mutation); diff --git a/src/instrument/counter.ts b/src/instrument/counter.ts deleted file mode 100644 index f5a68259..00000000 --- a/src/instrument/counter.ts +++ /dev/null @@ -1,46 +0,0 @@ -import {Method} from "../lib/enums"; - -interface ICounter { - [key: number]: ICounterValue; -} - -interface ICounterValue { - start: number; - end: number; - duration: number; - count: number; -} - -let tracker: ICounter = {}; -window["TRACKER"] = tracker; // DEBUG: Remove later -let threshold = 50; - -export function longtasks(method: Method): boolean { - let elapsed = Date.now() - tracker[method].start; - return (elapsed > threshold); -} - -export function start(method: Method): void { - if (!(method in tracker)) { - tracker[method] = { start: 0, end: 0, duration: 0, count: 0 }; - } - tracker[method].start = Date.now(); -} - -export function stop(method: Method): void { - tracker[method].end = Date.now(); - tracker[method].duration += tracker[method]["end"] - tracker[method]["start"]; - tracker[method].count++; -} - -export async function idle(method: Method): Promise { - stop(method); - await wait(); - start(method); -} - -async function wait(): Promise { - return new Promise((resolve: FrameRequestCallback): void => { - requestAnimationFrame(resolve); - }); -} diff --git a/src/lib/enums.ts b/src/lib/enums.ts deleted file mode 100644 index d43688fb..00000000 --- a/src/lib/enums.ts +++ /dev/null @@ -1,27 +0,0 @@ -export enum Method { - Discover, - Mutation, - Layout, - Serialize -} - -export enum Metric { - /* Core */ - Wireup, - Bytes, - /* DOM */ - DiscoverCount, - MutationCount, - /* Pointer */ - PointerDistance, - SwipeCount, - /* Viewport */ - ViewportX, - ViewportY, - ViewportWidth, - ViewportHeight, - DocumentWidth, - DocumentHeight, - ActiveTime, - TotalTime -} diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts index e4ec4ed6..fae1abe3 100644 --- a/src/lib/nodetree.ts +++ b/src/lib/nodetree.ts @@ -149,11 +149,12 @@ export class NodeTree { return nodes; } - public getValues(): INodeValue[] { + public summarize(): INodeValue[] { let values = []; for (let id in this.values) { - if (this.values[id] && this.values[id]["update"] === true) { + if (this.values[id].update) { values.push(this.values[id]); + this.values[id].update = false; } } return values; diff --git a/src/metrics/counter.ts b/src/metrics/counter.ts new file mode 100644 index 00000000..17a5cc47 --- /dev/null +++ b/src/metrics/counter.ts @@ -0,0 +1,38 @@ +import {Counter} from "./enums"; + +interface ICounter { + [key: number]: ICounterValue; +} + +interface ICounterValue { + updated: boolean; + counter: number; +} + +interface ICounterSummary { + [key: string]: ICounterSummaryValue; +} +interface ICounterSummaryValue { + counter: number; +} + +let tracker: ICounter = {}; +let summary: ICounterSummary = {}; + +export function increment(key: Counter, counter: number = 1): void { + if (!(key in tracker)) { + tracker[key] = { updated: true, counter }; + } + tracker[key].updated = true; + tracker[key].counter += counter; +} + +export function summarize(): ICounterSummary { + for (let key in tracker) { + if (tracker[key].updated) { + summary[key] = { counter: tracker[key].counter }; + tracker[key].updated = false; + } + } + return summary; +} diff --git a/src/metrics/enums.ts b/src/metrics/enums.ts new file mode 100644 index 00000000..2a3a92ec --- /dev/null +++ b/src/metrics/enums.ts @@ -0,0 +1,31 @@ +export enum Timer { + Discover = "dt", + Mutation = "mt", + Layout = "lt", + Serialize = "st", + Wireup = "wt", + Active = "at" +} + +export enum Counter { + Nodes = "nc", + Bytes = "bc", + Mutations = "mc", + Swipes = "sc", +} + +export enum Histogram { + PointerDistance = "ph", + ViewportX = "xh", + ViewportY = "yh", + ViewportWidth = "wh", + ViewportHeight = "hh", + DocumentWidth = "dwh", + DocumentHeight = "dhh" +} + +export enum Mark { + Click = "cm", + Error = "em", + Interaction = "ic" +} diff --git a/src/metrics/histogram.ts b/src/metrics/histogram.ts new file mode 100644 index 00000000..33800d81 --- /dev/null +++ b/src/metrics/histogram.ts @@ -0,0 +1,49 @@ +import {Histogram} from "./enums"; + +interface IHistogram { + [key: number]: IHistogramValue; +} + +interface IHistogramValue { + updated: boolean; + values: [number]; +} + +interface IHistogramSummary { + [key: string]: IHistogramSummaryValue; +} +interface IHistogramSummaryValue { + sum: number; + min: number; + max: number; + count: number; + sumsquared: number; +} + +let tracker: IHistogram = {}; +let summary: IHistogramSummary = {}; + +export function observe(key: Histogram, value: number): void { + if (!(key in tracker)) { + tracker[key] = { updated: true, values: [] }; + } + tracker[key].updated = true; + tracker[key].values.push(value); +} + +export function summarize(): IHistogramSummary { + for (let key in tracker) { + if (tracker[key].updated) { + summary[key] = { sum: 0, min: -1, max: -1, sumsquared: 0, count: 0 }; + for (let value of tracker[key].values) { + summary[key].sum += value; + summary[key].min = summary[key].count > 0 ? Math.min(summary[key].min, value) : value; + summary[key].max = summary[key].count > 0 ? Math.max(summary[key].max, value) : value; + summary[key].sumsquared += value; + summary[key].count++; + } + tracker[key].updated = false; + } + } + return summary; +} diff --git a/src/metrics/mark.ts b/src/metrics/mark.ts new file mode 100644 index 00000000..2e206602 --- /dev/null +++ b/src/metrics/mark.ts @@ -0,0 +1,32 @@ +import {Mark} from "./enums"; + +interface IMark { + mark: string; + updated: boolean; + start: number; + end: number; +} + +interface IMarkSummary { + mark: string; + start: number; + end: number; +} + +let tracker: IMark[] = []; +let summary: IMarkSummary[] = []; + +export function mark(key: Mark, tag: string, start: number, end: number = 0): void { + end = end > 0 ? end : start; + tracker.push( { updated: true, mark: tag, start, end}); +} + +export function summarize(): IMarkSummary[] { + for (let entry of tracker) { + if (entry.updated) { + summary.push({mark: entry.mark, start: entry.start, end: entry.end }); + entry.updated = false; + } + } + return summary; +} diff --git a/src/metrics/timer.ts b/src/metrics/timer.ts new file mode 100644 index 00000000..88cf37f7 --- /dev/null +++ b/src/metrics/timer.ts @@ -0,0 +1,67 @@ +import {Timer} from "./enums"; + +interface ITimer { + [key: number]: ITimerValue; +} + +interface ITimerValue { + updated: boolean; + start: number; + end: number; + duration: number; + count: number; +} + +interface ITimerSummary { + [key: number]: ITimerSummaryValue; +} + +interface ITimerSummaryValue { + duration: number; + count: number; +} + +let tracker: ITimer = {}; +let summary: ITimerSummary = {}; +let threshold = 50; +window["TRACKER"] = tracker; // DEBUG: Remove later + +export function longtasks(method: Timer): boolean { + let elapsed = Date.now() - tracker[method].start; + return (elapsed > threshold); +} + +export function start(method: Timer): void { + if (!(method in tracker)) { + tracker[method] = { start: 0, end: 0, duration: 0, count: 0 }; + } + tracker[method].start = Date.now(); +} + +export function stop(method: Timer): void { + tracker[method].end = Date.now(); + tracker[method].duration += tracker[method]["end"] - tracker[method]["start"]; + tracker[method].count++; +} + +export async function idle(method: Timer): Promise { + stop(method); + await wait(); + start(method); +} + +async function wait(): Promise { + return new Promise((resolve: FrameRequestCallback): void => { + requestAnimationFrame(resolve); + }); +} + +export function summarize(): ITimerSummary { + for (let key in tracker) { + if (tracker[key].updated) { + summary[key] = { duration: tracker[key].duration, count: tracker[key].count }; + tracker[key].updated = false; + } + } + return summary; +} From 80e9a4209b038c3de8d496340468acc429675596 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 11 Jul 2019 08:03:23 -0700 Subject: [PATCH 011/105] Adding basic support for types and serialization --- src/core.ts | 6 +- src/data/deserialization/dom.ts | 148 ++++++++++++++++++++++++++++++ src/data/deserialize.ts | 148 +----------------------------- src/data/serialization/dom.ts | 14 ++- src/data/serialization/metrics.ts | 51 ++++++++++ src/data/serialize.ts | 16 ++++ src/lib/nodetree.ts | 21 +---- src/metrics/counter.ts | 17 +--- src/metrics/histogram.ts | 21 +---- src/metrics/mark.ts | 14 +-- src/metrics/timer.ts | 22 +---- types/data.d.ts | 1 + types/dom.d.ts | 20 ++++ types/index.d.ts | 4 + types/metrics.d.ts | 75 +++++++++++++++ 15 files changed, 332 insertions(+), 246 deletions(-) create mode 100644 src/data/deserialization/dom.ts create mode 100644 src/data/serialize.ts create mode 100644 types/data.d.ts create mode 100644 types/dom.d.ts create mode 100644 types/metrics.d.ts diff --git a/src/core.ts b/src/core.ts index df884547..49e6a155 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,5 +1,5 @@ import deserialize from "./data/deserialize"; -import serialize from "./data/serialization/dom"; +import serialize from "./data/serialize"; import discover from "./dom/discover"; import mutation from "./dom/mutation"; @@ -12,13 +12,13 @@ export function init(): void { discover().then(() => { // DEBUG: Remove later console.log("done discovery!"); - console.log(window["TRACKER"][0]["duration"] + "ms in " + window["TRACKER"][0]["count"] + " iterations"); + console.log(window["TRACKER"]["dt"]["duration"] + "ms in " + window["TRACKER"]["dt"]["count"] + " iterations"); // DEBUG: Serialize DOM serialize().then((output: string) => { console.log("Serialized DOM: " + output); console.log("Serialized DOM Length: " + output.length); console.log("done serialization!"); - console.log(window["TRACKER"][3]["duration"] + "ms in " + window["TRACKER"][3]["count"] + " iterations"); + console.log(window["TRACKER"]["st"]["duration"] + "ms in " + window["TRACKER"]["st"]["count"] + " iterations"); console.log("===================="); let deserialized = deserialize(output); console.log("Deserialized DOM: " + deserialized); diff --git a/src/data/deserialization/dom.ts b/src/data/deserialization/dom.ts new file mode 100644 index 00000000..1d2f775c --- /dev/null +++ b/src/data/deserialization/dom.ts @@ -0,0 +1,148 @@ +import { Token } from "@clarity-types/data"; +import { resolve } from "@src/data/token"; + +let nodes = {}; +let placeholder = document.createElement("iframe"); +// debug: visualize it on page +placeholder.height = "350"; +document.body.appendChild(placeholder); + +export default function(tokens: Token[]): string { + let number = 0; + let lastType = null; + let node = []; + let intermediate = []; + let tagIndex = 0; + for (let token of tokens) { + let type = typeof(token); + switch (type) { + case "number": + if (type !== lastType && lastType !== null) { + process(node, tagIndex); + intermediate.push(...node); + node = []; + tagIndex = 0; + } + number += token as number; + token = token === 0 ? token : number; + node.push(token); + tagIndex++; + break; + case "string": + node.push(token); + break; + case "object": + let subtoken = token[0]; + let subtype = typeof(subtoken); + switch (subtype) { + case "string": + let keys = resolve(token as string); + for (let key of keys) { + node.push(key); + } + break; + case "number": + token = tokens.length > subtoken ? tokens[subtoken] : null; + node.push(token); + break; + } + } + lastType = type; + } + + let markup = placeholder.contentDocument.documentElement.outerHTML; + + return JSON.stringify(intermediate) + "\r\n" + markup; +} + +function process(node: any[] | number[], tagIndex: number): void { + let id = node[0]; + let parent = tagIndex > 1 ? element(node[1]) : null; + let next = tagIndex > 2 ? element(node[2]) : null; + let layouts = []; + let tag = node[tagIndex]; + let doc = placeholder.contentDocument; + let content = null; + let attributes = {}; + + for (let i = tagIndex + 1; i < node.length; i++) { + let token = node[i] as string; + let keyIndex = token.indexOf("="); + let parts = token.split("*"); + if (tag !== "*T" && keyIndex > 0) { + attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); + } else if (parts.length === 4) { + let layout = []; + for (let part of parts) { + layout.push(parseInt(part, 36)); + } + layouts.push(layout); + } else if (tag === "*T" && parts.length === 2) { + let textCount = parseInt(parts[0], 36); + let wordCount = parseInt(parts[1], 36); + content = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); + } else if (tag === "*T") { + content = token; + } + } + + switch (tag) { + case "*D": + if (typeof XMLSerializer !== "undefined" && false) { + doc.open(); + doc.write(new XMLSerializer().serializeToString( + doc.implementation.createDocumentType( + attributes["name"], + attributes["publicId"], + attributes["systemId"] + ) + )); + doc.close(); + } + break; + case "*T": + let textElement = element(id); + textElement = textElement ? textElement : doc.createTextNode(null); + textElement.nodeValue = content; + insert(id, parent, textElement, next); + break; + case "HTML": + let newDoc = doc.implementation.createHTMLDocument(""); + let html = newDoc.documentElement; + let pointer = doc.importNode(html, true); + doc.replaceChild(pointer, doc.documentElement); + if (doc.head) { doc.head.parentNode.removeChild(doc.head); } + if (doc.body) { doc.body.parentNode.removeChild(doc.body); } + nodes[id] = doc.documentElement; + break; + default: + let domElement = element(id); + domElement = domElement ? domElement : doc.createElement(tag); + attributes["data-id"] = id; + setAttributes(domElement as HTMLElement, attributes); + insert(id, parent, domElement, next); + break; + } +} + +function element(nodeId: number): Node { + return nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; +} + +function insert(id: number, parent: Node, node: Node, next: Node): void { + parent.insertBefore(node, next); + nodes[id] = node; +} + +function setAttributes(node: HTMLElement, attributes: object): void { + for (let attribute in node.attributes) { + if (node.hasAttribute(attribute)) { + node.removeAttribute(attribute); + } + } + for (let attribute in attributes) { + if (attributes[attribute]) { + node.setAttribute(attribute, attributes[attribute]); + } + } +} diff --git a/src/data/deserialize.ts b/src/data/deserialize.ts index 1db63552..d296855c 100644 --- a/src/data/deserialize.ts +++ b/src/data/deserialize.ts @@ -1,148 +1,6 @@ -import { resolve } from "@src/data/token"; - -let nodes = {}; -let placeholder = document.createElement("iframe"); -// debug: visualize it on page -placeholder.height = "350"; -document.body.appendChild(placeholder); +import domDeserialize from "./deserialization/dom"; export default function(payload: string): string { - let tokens = JSON.parse(payload); - let number = 0; - let lastType = null; - let node = []; - let intermediate = []; - let tagIndex = 0; - for (let token of tokens) { - let type = typeof(token); - switch (type) { - case "number": - if (type !== lastType && lastType !== null) { - process(node, tagIndex); - intermediate.push(...node); - node = []; - tagIndex = 0; - } - number += token; - token = token === 0 ? token : number; - node.push(token); - tagIndex++; - break; - case "string": - node.push(token); - break; - case "object": - let subtoken = token[0]; - let subtype = typeof(subtoken); - switch (subtype) { - case "string": - let keys = resolve(token); - for (let key of keys) { - node.push(key); - } - break; - case "number": - token = tokens.length > subtoken ? tokens[subtoken] : null; - node.push(token); - break; - } - } - lastType = type; - } - - let markup = placeholder.contentDocument.documentElement.outerHTML; - - return JSON.stringify(intermediate) + "\r\n" + markup; -} - -function process(node: any[] | number[], tagIndex: number): void { - let id = node[0]; - let parent = tagIndex > 1 ? element(node[1]) : null; - let next = tagIndex > 2 ? element(node[2]) : null; - let layouts = []; - let tag = node[tagIndex]; - let doc = placeholder.contentDocument; - let content = null; - let attributes = {}; - - for (let i = tagIndex + 1; i < node.length; i++) { - let token = node[i] as string; - let keyIndex = token.indexOf("="); - let parts = token.split("*"); - if (tag !== "*T" && keyIndex > 0) { - attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); - } else if (parts.length === 4) { - let layout = []; - for (let part of parts) { - layout.push(parseInt(part, 36)); - } - layouts.push(layout); - } else if (tag === "*T" && parts.length === 2) { - let textCount = parseInt(parts[0], 36); - let wordCount = parseInt(parts[1], 36); - content = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); - } else if (tag === "*T") { - content = token; - } - } - - switch (tag) { - case "*D": - if (typeof XMLSerializer !== "undefined" && false) { - doc.open(); - doc.write(new XMLSerializer().serializeToString( - doc.implementation.createDocumentType( - attributes["name"], - attributes["publicId"], - attributes["systemId"] - ) - )); - doc.close(); - } - break; - case "*T": - let textElement = element(id); - textElement = textElement ? textElement : doc.createTextNode(null); - textElement.nodeValue = content; - insert(id, parent, textElement, next); - break; - case "HTML": - let newDoc = doc.implementation.createHTMLDocument(""); - let html = newDoc.documentElement; - let pointer = doc.importNode(html, true); - doc.replaceChild(pointer, doc.documentElement); - if (doc.head) { doc.head.parentNode.removeChild(doc.head); } - if (doc.body) { doc.body.parentNode.removeChild(doc.body); } - nodes[id] = doc.documentElement; - break; - default: - let domElement = element(id); - domElement = domElement ? domElement : doc.createElement(tag); - attributes["data-id"] = id; - setAttributes(domElement as HTMLElement, attributes); - insert(id, parent, domElement, next); - break; - } -} - -function element(nodeId: number): Node { - return nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; -} - -function insert(id: number, parent: Node, node: Node, next: Node): void { - parent.insertBefore(node, next); - nodes[id] = node; -} - -function setAttributes(node: HTMLElement, attributes: object): void { - for (let attribute in node.attributes) { - if (node.hasAttribute(attribute)) { - node.removeAttribute(attribute); - } - } - for (let attribute in attributes) { - if (attributes[attribute]) { - node.setAttribute(attribute, attributes[attribute]); - } - } + let json = JSON.parse(payload); + return domDeserialize(json.dom); } diff --git a/src/data/serialization/dom.ts b/src/data/serialization/dom.ts index 9c2897b0..0a2689eb 100644 --- a/src/data/serialization/dom.ts +++ b/src/data/serialization/dom.ts @@ -1,5 +1,6 @@ +import {Token} from "@clarity-types/data"; +import {INodeData} from "@clarity-types/dom"; import hash from "../../lib/hash"; -import {INodeData} from "../../lib/nodetree"; import * as counter from "../../metrics/counter"; import { Counter, Timer } from "../../metrics/enums"; import * as timer from "../../metrics/timer"; @@ -9,14 +10,13 @@ import {check} from "../token"; window["HASH"] = hash; let reference: number = 0; -export default async function(): Promise { - let method = Timer.Serialize; - timer.start(method); +export default async function(): Promise { + let tracker = Timer.Serialize; let markup = []; let values = nodes.summarize(); reference = 0; for (let value of values) { - if (timer.longtasks(method)) { await timer.idle(method); } + if (timer.longtasks(tracker)) { await timer.idle(tracker); } let metadata = []; let layouts = []; let data: INodeData = value.data; @@ -66,9 +66,7 @@ export default async function(): Promise { markup.push(entry); } } - let json = JSON.stringify(markup); - timer.stop(method); - return json; + return markup; } function meta(metadata: string[]): string[] | string[][] { diff --git a/src/data/serialization/metrics.ts b/src/data/serialization/metrics.ts index e69de29b..36fde806 100644 --- a/src/data/serialization/metrics.ts +++ b/src/data/serialization/metrics.ts @@ -0,0 +1,51 @@ +import {Token} from "@clarity-types/data"; +import * as counter from "../../metrics/counter"; +import { Timer } from "../../metrics/enums"; +import * as histogram from "../../metrics/histogram"; +import * as mark from "../../metrics/mark"; +import * as timer from "../../metrics/timer"; + +export default async function(): Promise { + let tracker = Timer.Serialize; + let metrics = []; + if (timer.longtasks(tracker)) { await timer.idle(tracker); } + + // Serialize counters + let counters = counter.summarize(); + for (let key in counters) { + if (counters[key]) { + let c = counters[key]; + metrics.push([key, num(c.counter)].join("*")); + } + } + + // Serialize histograms + let histograms = histogram.summarize(); + for (let key in histograms) { + if (histograms[key]) { + let h = histograms[key]; + metrics.push([key, num(h.sum), num(h.min), num(h.max), num(h.sumsquared), num(h.count)].join("*")); + } + } + + // Serialize timers + let timers = timer.summarize(); + for (let key in timers) { + if (timers[key]) { + let t = timers[key]; + metrics.push([key, num(t.duration), num(t.count)].join("*")); + } + } + + // Serialize marks + let marks = mark.summarize(); + for (let m of marks) { + metrics.push([m.mark, num(m.start), num(m.end)].join("*")); + } + + return metrics; +} + +function num(x: number): string { + return x.toString(36); +} diff --git a/src/data/serialize.ts b/src/data/serialize.ts new file mode 100644 index 00000000..0949ffdd --- /dev/null +++ b/src/data/serialize.ts @@ -0,0 +1,16 @@ +import { Timer } from "../metrics/enums"; +import * as timer from "../metrics/timer"; +import serializeDOM from "./serialization/dom"; +import serializeMetrics from "./serialization/metrics"; + +export default async function(): Promise { + let tracker = Timer.Serialize; + timer.start(tracker); + let json = { + dom: await serializeDOM(), + metrics: await serializeMetrics() + }; + let output = JSON.stringify(json); + timer.stop(tracker); + return output; +} diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts index fae1abe3..25d5b3e3 100644 --- a/src/lib/nodetree.ts +++ b/src/lib/nodetree.ts @@ -1,23 +1,4 @@ -export interface IAttributes { - [key: string]: string; -} - -export interface INodeData { - tag: string; - attributes?: IAttributes; - layout?: number[]; - value?: string; -} - -export interface INodeValue { - id: number; - parent: number; - next: number; - children: number[]; - data: INodeData; - active?: boolean; - update?: boolean; -} +import {INodeData, INodeValue } from "@clarity-types/dom"; export class NodeTree { diff --git a/src/metrics/counter.ts b/src/metrics/counter.ts index 17a5cc47..fe76b50d 100644 --- a/src/metrics/counter.ts +++ b/src/metrics/counter.ts @@ -1,21 +1,6 @@ +import {ICounter, ICounterSummary} from "@clarity-types/metrics"; import {Counter} from "./enums"; -interface ICounter { - [key: number]: ICounterValue; -} - -interface ICounterValue { - updated: boolean; - counter: number; -} - -interface ICounterSummary { - [key: string]: ICounterSummaryValue; -} -interface ICounterSummaryValue { - counter: number; -} - let tracker: ICounter = {}; let summary: ICounterSummary = {}; diff --git a/src/metrics/histogram.ts b/src/metrics/histogram.ts index 33800d81..31a4b701 100644 --- a/src/metrics/histogram.ts +++ b/src/metrics/histogram.ts @@ -1,25 +1,6 @@ +import {IHistogram, IHistogramSummary} from "@clarity-types/metrics"; import {Histogram} from "./enums"; -interface IHistogram { - [key: number]: IHistogramValue; -} - -interface IHistogramValue { - updated: boolean; - values: [number]; -} - -interface IHistogramSummary { - [key: string]: IHistogramSummaryValue; -} -interface IHistogramSummaryValue { - sum: number; - min: number; - max: number; - count: number; - sumsquared: number; -} - let tracker: IHistogram = {}; let summary: IHistogramSummary = {}; diff --git a/src/metrics/mark.ts b/src/metrics/mark.ts index 2e206602..3ac1d8f0 100644 --- a/src/metrics/mark.ts +++ b/src/metrics/mark.ts @@ -1,18 +1,6 @@ +import {IMark, IMarkSummary} from "@clarity-types/metrics"; import {Mark} from "./enums"; -interface IMark { - mark: string; - updated: boolean; - start: number; - end: number; -} - -interface IMarkSummary { - mark: string; - start: number; - end: number; -} - let tracker: IMark[] = []; let summary: IMarkSummary[] = []; diff --git a/src/metrics/timer.ts b/src/metrics/timer.ts index 88cf37f7..884b7d96 100644 --- a/src/metrics/timer.ts +++ b/src/metrics/timer.ts @@ -1,26 +1,6 @@ +import {ITimer, ITimerSummary} from "@clarity-types/metrics"; import {Timer} from "./enums"; -interface ITimer { - [key: number]: ITimerValue; -} - -interface ITimerValue { - updated: boolean; - start: number; - end: number; - duration: number; - count: number; -} - -interface ITimerSummary { - [key: number]: ITimerSummaryValue; -} - -interface ITimerSummaryValue { - duration: number; - count: number; -} - let tracker: ITimer = {}; let summary: ITimerSummary = {}; let threshold = 50; diff --git a/types/data.d.ts b/types/data.d.ts new file mode 100644 index 00000000..3bfad016 --- /dev/null +++ b/types/data.d.ts @@ -0,0 +1 @@ +export type Token = (string | number | number[] | string[]); diff --git a/types/dom.d.ts b/types/dom.d.ts new file mode 100644 index 00000000..22bfe622 --- /dev/null +++ b/types/dom.d.ts @@ -0,0 +1,20 @@ +export interface IAttributes { + [key: string]: string; +} + +export interface INodeData { + tag: string; + attributes?: IAttributes; + layout?: number[]; + value?: string; +} + +export interface INodeValue { + id: number; + parent: number; + next: number; + children: number[]; + data: INodeData; + active?: boolean; + update?: boolean; +} diff --git a/types/index.d.ts b/types/index.d.ts index 8b741fb2..60c1c2ff 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -3,3 +3,7 @@ interface IClarityJs { } declare const ClarityJs: IClarityJs; + +export * from "./data"; +export * from "./dom"; +export * from "./metrics"; diff --git a/types/metrics.d.ts b/types/metrics.d.ts new file mode 100644 index 00000000..689794d1 --- /dev/null +++ b/types/metrics.d.ts @@ -0,0 +1,75 @@ +// Counter +export interface ICounter { + [key: number]: ICounterValue; +} + +interface ICounterValue { + updated: boolean; + counter: number; +} + +export interface ICounterSummary { + [key: string]: ICounterSummaryValue; +} + +interface ICounterSummaryValue { + counter: number; +} + +// Histogram +export interface IHistogram { + [key: number]: IHistogramValue; +} + +interface IHistogramValue { + updated: boolean; + values: [number]; +} + +export interface IHistogramSummary { + [key: string]: IHistogramSummaryValue; +} + +interface IHistogramSummaryValue { + sum: number; + min: number; + max: number; + count: number; + sumsquared: number; +} + +// Mark +export interface IMark { + mark: string; + updated: boolean; + start: number; + end: number; +} + +export interface IMarkSummary { + mark: string; + start: number; + end: number; +} + +// Timer +export interface ITimer { + [key: number]: ITimerValue; +} + +interface ITimerValue { + updated: boolean; + start: number; + end: number; + duration: number; + count: number; +} + +export interface ITimerSummary { + [key: number]: ITimerSummaryValue; +} + +interface ITimerSummaryValue { + duration: number; + count: number; +} From 0de53786d9e4a35651f46deea55ef7abbfcba08a Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 12 Jul 2019 08:10:22 -0700 Subject: [PATCH 012/105] Restructuring files --- src/core.ts | 2 +- src/data/deserialize.ts | 2 +- src/data/{serialize.ts => send.ts} | 4 ++-- .../deserialization/dom.ts => dom/deserialize.ts} | 0 src/{data/serialization/dom.ts => dom/serialize.ts} | 12 ++++++------ .../metrics.ts => metrics/serialize.ts} | 10 +++++----- 6 files changed, 15 insertions(+), 15 deletions(-) rename src/data/{serialize.ts => send.ts} (78%) rename src/{data/deserialization/dom.ts => dom/deserialize.ts} (100%) rename src/{data/serialization/dom.ts => dom/serialize.ts} (93%) rename src/{data/serialization/metrics.ts => metrics/serialize.ts} (83%) diff --git a/src/core.ts b/src/core.ts index 49e6a155..c01c0e33 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,5 +1,5 @@ import deserialize from "./data/deserialize"; -import serialize from "./data/serialize"; +import serialize from "./data/send"; import discover from "./dom/discover"; import mutation from "./dom/mutation"; diff --git a/src/data/deserialize.ts b/src/data/deserialize.ts index d296855c..fd9f428b 100644 --- a/src/data/deserialize.ts +++ b/src/data/deserialize.ts @@ -1,4 +1,4 @@ -import domDeserialize from "./deserialization/dom"; +import domDeserialize from "../dom/deserialize"; export default function(payload: string): string { let json = JSON.parse(payload); diff --git a/src/data/serialize.ts b/src/data/send.ts similarity index 78% rename from src/data/serialize.ts rename to src/data/send.ts index 0949ffdd..0626902f 100644 --- a/src/data/serialize.ts +++ b/src/data/send.ts @@ -1,7 +1,7 @@ +import serializeDOM from "../dom/serialize"; import { Timer } from "../metrics/enums"; +import serializeMetrics from "../metrics/serialize"; import * as timer from "../metrics/timer"; -import serializeDOM from "./serialization/dom"; -import serializeMetrics from "./serialization/metrics"; export default async function(): Promise { let tracker = Timer.Serialize; diff --git a/src/data/deserialization/dom.ts b/src/dom/deserialize.ts similarity index 100% rename from src/data/deserialization/dom.ts rename to src/dom/deserialize.ts diff --git a/src/data/serialization/dom.ts b/src/dom/serialize.ts similarity index 93% rename from src/data/serialization/dom.ts rename to src/dom/serialize.ts index 0a2689eb..bd632bff 100644 --- a/src/data/serialization/dom.ts +++ b/src/dom/serialize.ts @@ -1,11 +1,11 @@ import {Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; -import hash from "../../lib/hash"; -import * as counter from "../../metrics/counter"; -import { Counter, Timer } from "../../metrics/enums"; -import * as timer from "../../metrics/timer"; -import {nodes} from "../state"; -import {check} from "../token"; +import {nodes} from "../data/state"; +import {check} from "../data/token"; +import hash from "../lib/hash"; +import * as counter from "../metrics/counter"; +import { Counter, Timer } from "../metrics/enums"; +import * as timer from "../metrics/timer"; window["HASH"] = hash; let reference: number = 0; diff --git a/src/data/serialization/metrics.ts b/src/metrics/serialize.ts similarity index 83% rename from src/data/serialization/metrics.ts rename to src/metrics/serialize.ts index 36fde806..457db00b 100644 --- a/src/data/serialization/metrics.ts +++ b/src/metrics/serialize.ts @@ -1,9 +1,9 @@ import {Token} from "@clarity-types/data"; -import * as counter from "../../metrics/counter"; -import { Timer } from "../../metrics/enums"; -import * as histogram from "../../metrics/histogram"; -import * as mark from "../../metrics/mark"; -import * as timer from "../../metrics/timer"; +import * as counter from "./counter"; +import { Timer } from "./enums"; +import * as histogram from "./histogram"; +import * as mark from "./mark"; +import * as timer from "./timer"; export default async function(): Promise { let tracker = Timer.Serialize; From d01cef9add516d37fd839cb1963a02c0356b76eb Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 12 Jul 2019 18:39:05 -0700 Subject: [PATCH 013/105] Restructing files --- src/{lib => data}/hash.ts | 0 src/data/state.ts | 4 - src/dom/node.ts | 2 +- src/dom/serialize.ts | 6 +- src/dom/virtualdom.ts | 156 ++++++++++++++++++++++++++++++++++++ src/lib/nodetree.ts | 164 -------------------------------------- 6 files changed, 160 insertions(+), 172 deletions(-) rename src/{lib => data}/hash.ts (100%) delete mode 100644 src/data/state.ts create mode 100644 src/dom/virtualdom.ts delete mode 100644 src/lib/nodetree.ts diff --git a/src/lib/hash.ts b/src/data/hash.ts similarity index 100% rename from src/lib/hash.ts rename to src/data/hash.ts diff --git a/src/data/state.ts b/src/data/state.ts deleted file mode 100644 index 19bb56f5..00000000 --- a/src/data/state.ts +++ /dev/null @@ -1,4 +0,0 @@ -import {NodeTree} from "../lib/nodetree"; - -export let nodes = new NodeTree(); -window["NODES"] = nodes; // DEBUG: Remove later diff --git a/src/dom/node.ts b/src/dom/node.ts index 685ca012..7af853e1 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -1,4 +1,4 @@ -import {nodes} from "../data/state"; +import * as nodes from "./virtualdom"; let ignoreAttributes = ["title", "alt", "onload", "onfocus"]; diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts index bd632bff..a91d8f9d 100644 --- a/src/dom/serialize.ts +++ b/src/dom/serialize.ts @@ -1,11 +1,11 @@ import {Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; -import {nodes} from "../data/state"; +import hash from "../data/hash"; import {check} from "../data/token"; -import hash from "../lib/hash"; import * as counter from "../metrics/counter"; import { Counter, Timer } from "../metrics/enums"; import * as timer from "../metrics/timer"; +import * as nodes from "./virtualdom"; window["HASH"] = hash; let reference: number = 0; @@ -47,7 +47,7 @@ export default async function(): Promise { } break; case "value": - let parent = nodes.node(value.parent); + let parent = nodes.getNode(value.parent); let parentTag = nodes.get(parent).data.tag; metadata.push(text(parentTag, data[key])); break; diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts new file mode 100644 index 00000000..87a77613 --- /dev/null +++ b/src/dom/virtualdom.ts @@ -0,0 +1,156 @@ +import {INodeData, INodeValue } from "@clarity-types/dom"; + +const NODE_ID_PROP: string = "__node_index__"; +let index: number = 1; + +let nodes: Node[] = []; +let values: INodeValue[] = []; + +let backupIndex: number; +let backupNodes: Node[]; +let backupValues: Node[]; + +export function getId(node: Node, autogen: boolean = true): number { + if (node === null) { return null; } + let id = node[NODE_ID_PROP]; + if (!id && autogen) { + id = node[NODE_ID_PROP] = index++; + } + return id ? id : null; +} + +export function add(node: Node, data: INodeData): void { + let id = getId(node); + let parentId = node.parentElement ? getId(node.parentElement) : null; + let nextId = node.nextSibling ? getId(node.nextSibling, false) : null; + + if (parentId >= 0 && values[parentId]) { + values[parentId].children.push(id); + } + + nodes[id] = node; + values[id] = { + id, + parent: parentId, + next: nextId, + children: [], + active: true, + update: true, + data + }; +} + +export function update(node: Node, data: INodeData): void { + let id = getId(node); + console.log("Updating node: " + id); + let parentId = node.parentElement ? getId(node.parentElement) : null; + let nextId = node.nextSibling ? getId(node.nextSibling) : null; + + if (id in values) { + let value = values[id]; + console.log("Previous value: " + JSON.stringify(value)); + + // Handle case where internal ordering may have changed + if (value["next"] !== nextId) { + let oldNextId = value["next"]; + value["next"] = nextId; + console.log("Old next id: " + oldNextId + " | " + nextId); + } + + // Handle case where parent might have been updated + if (value["parent"] !== parentId) { + let oldParentId = value["parent"]; + value["parent"] = parentId; + console.log("Old parent id: " + oldParentId + " | " + parentId); + // Move this node to the right location under new parent + if (parentId >= 0) { + if (nextId >= 0) { + values[parentId].children.splice(nextId + 1, 0 , id); + } else { + values[parentId].children.push(id); + } + } else { + // Mark this element as deleted if the parent has been updated to null + value["active"] = false; + } + + // Remove reference to this node from the old parent + let nodeIndex = values[oldParentId].children.indexOf(id); + if (nodeIndex >= 0) { + values[oldParentId].children.splice(nodeIndex, 1); + } + } + + // Update data + for (let key in data) { + if (key in value["data"]) { + value["data"][key] = data[key]; + } + } + + value["update"] = true; + } +} + +export function getNode(id: number): Node { + if (id in nodes) { + return nodes[id]; + } + return null; +} + +export function get(node: Node): INodeValue { + let id = getId(node); + return values[id]; +} + +export function has(node: Node): boolean { + return getId(node) in nodes; +} + +export function remove(node: Node): void { + let id = getId(node); + del(id); +} + +export function getNodes(): Node[] { + let n: Node[] = []; + for (let id in nodes) { + if (nodes[id]) { + n.push(nodes[id]); + } + } + return n; +} + +export function summarize(): INodeValue[] { + let v = []; + for (let id in values) { + if (values[id].update) { + values[id].update = false; + v.push(values[id]); + } + } + return v; +} + +export function backup(): void { + backupNodes = Array.from(nodes); + backupValues = JSON.parse(JSON.stringify(values)); + backupIndex = index; +} + +export function rollback(): void { + nodes = Array.from(backupNodes); + values = JSON.parse(JSON.stringify(backupValues)); + index = backupIndex; +} + +function del(id: number): void { + let children = values[id].children; + for (let i = 0; i < children.length; i++) { + del(children[i]); + } + values[id].active = false; + values[id].update = true; +} diff --git a/src/lib/nodetree.ts b/src/lib/nodetree.ts deleted file mode 100644 index 25d5b3e3..00000000 --- a/src/lib/nodetree.ts +++ /dev/null @@ -1,164 +0,0 @@ -import {INodeData, INodeValue } from "@clarity-types/dom"; - -export class NodeTree { - - private static NODE_ID_PROP: string = "__node_index__"; - private static index: number = 1; - - private nodes: Node[]; - private values: INodeValue[]; - - private backupIndex: number; - private backupNodes: Node[]; - private backupValues: Node[]; - - constructor() { - this.nodes = []; - this.values = []; - } - - public id(node: Node, autogen: boolean = true): number { - if (node === null) { return null; } - let id = node[NodeTree.NODE_ID_PROP]; - if (!id && autogen) { - id = node[NodeTree.NODE_ID_PROP] = NodeTree.index++; - } - return id ? id : null; - } - - public add(node: Node, data: INodeData): void { - let id = this.id(node); - let parentId = node.parentElement ? this.id(node.parentElement) : null; - let nextId = node.nextSibling ? this.id(node.nextSibling, false) : null; - - if (parentId >= 0 && this.values[parentId]) { - this.values[parentId].children.push(id); - } - - this.nodes[id] = node; - this.values[id] = { - id, - parent: parentId, - next: nextId, - children: [], - active: true, - update: true, - data - }; - } - - public update(node: Node, data: INodeData): void { - let id = this.id(node); - console.log("Updating node: " + id); - let parentId = node.parentElement ? this.id(node.parentElement) : null; - let nextId = node.nextSibling ? this.id(node.nextSibling) : null; - - if (id in this.values) { - let value = this.values[id]; - console.log("Previous value: " + JSON.stringify(value)); - - // Handle case where internal ordering may have changed - if (value["next"] !== nextId) { - let oldNextId = value["next"]; - value["next"] = nextId; - console.log("Old next id: " + oldNextId + " | " + nextId); - } - - // Handle case where parent might have been updated - if (value["parent"] !== parentId) { - let oldParentId = value["parent"]; - value["parent"] = parentId; - console.log("Old parent id: " + oldParentId + " | " + parentId); - // Move this node to the right location under new parent - if (parentId >= 0) { - if (nextId >= 0) { - this.values[parentId].children.splice(nextId + 1, 0 , id); - } else { - this.values[parentId].children.push(id); - } - } else { - // Mark this element as deleted if the parent has been updated to null - value["active"] = false; - } - - // Remove reference to this node from the old parent - let index = this.values[oldParentId].children.indexOf(id); - if (index >= 0) { - this.values[oldParentId].children.splice(index, 1); - } - } - - // Update data - for (let key in data) { - if (key in value["data"]) { - value["data"][key] = data[key]; - } - } - - value["update"] = true; - } - } - - public node(id: number): Node { - if (id in this.nodes) { - return this.nodes[id]; - } - return null; - } - - public get(node: Node): INodeValue { - let id = this.id(node); - return this.values[id]; - } - - public has(node: Node): boolean { - return this.id(node) in this.nodes; - } - - public remove(node: Node): void { - let id = this.id(node); - this.delete(id); - } - - public getNodes(): Node[] { - let nodes: Node[] = []; - for (let id in this.nodes) { - if (this.nodes[id]) { - nodes.push(this.nodes[id]); - } - } - return nodes; - } - - public summarize(): INodeValue[] { - let values = []; - for (let id in this.values) { - if (this.values[id].update) { - values.push(this.values[id]); - this.values[id].update = false; - } - } - return values; - } - - public backup(): void { - this.backupNodes = Array.from(this.nodes); - this.backupValues = JSON.parse(JSON.stringify(this.values)); - this.backupIndex = NodeTree.index; - } - - public rollback(): void { - this.nodes = Array.from(this.backupNodes); - this.values = JSON.parse(JSON.stringify(this.backupValues)); - NodeTree.index = this.backupIndex; - } - - private delete(id: number): void { - let children = this.values[id].children; - for (let i = 0; i < children.length; i++) { - this.delete(children[i]); - } - this.values[id].active = false; - this.values[id].update = true; - } -} From af71719a2d79a1302d987523887a2cbc21f618dc Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 15 Jul 2019 08:12:27 -0700 Subject: [PATCH 014/105] Adding support for viewport and mouse interactions --- src/core.ts | 52 ++++++++++++--------- src/data/deserialize.ts | 2 +- src/data/send.ts | 16 ------- src/data/upload.ts | 38 ++++++++++++++++ src/dom/discover.ts | 18 ++++++-- src/dom/mutation.ts | 85 +++++++++++++++++++---------------- src/dom/serialize.ts | 15 +++---- src/interactions/keyboard.ts | 0 src/interactions/mouse.ts | 66 +++++++++++++++++++++++++++ src/interactions/selection.ts | 0 src/interactions/serialize.ts | 25 +++++++++++ src/interactions/touch.ts | 0 src/metrics/counter.ts | 3 +- src/metrics/enums.ts | 31 ------------- src/metrics/histogram.ts | 3 +- src/metrics/mark.ts | 3 +- src/metrics/metrics.ts | 8 ++++ src/metrics/serialize.ts | 5 +-- src/metrics/timer.ts | 3 +- src/performance/navigation.ts | 0 src/performance/resource.ts | 0 src/viewport/document.ts | 40 +++++++++++++++++ src/viewport/resize.ts | 25 +++++++++++ src/viewport/scroll.ts | 55 +++++++++++++++++++++++ src/viewport/serialize.ts | 33 ++++++++++++++ src/viewport/visibility.ts | 26 +++++++++++ types/core.d.ts | 8 +++- types/data.d.ts | 28 ++++++++++++ types/interactions.d.ts | 17 +++++++ types/metrics.d.ts | 31 +++++++++++++ types/viewport.d.ts | 23 ++++++++++ 31 files changed, 529 insertions(+), 130 deletions(-) delete mode 100644 src/data/send.ts create mode 100644 src/data/upload.ts create mode 100644 src/interactions/keyboard.ts create mode 100644 src/interactions/mouse.ts create mode 100644 src/interactions/selection.ts create mode 100644 src/interactions/serialize.ts create mode 100644 src/interactions/touch.ts delete mode 100644 src/metrics/enums.ts create mode 100644 src/metrics/metrics.ts create mode 100644 src/performance/navigation.ts create mode 100644 src/performance/resource.ts create mode 100644 src/viewport/document.ts create mode 100644 src/viewport/resize.ts create mode 100644 src/viewport/scroll.ts create mode 100644 src/viewport/serialize.ts create mode 100644 src/viewport/visibility.ts create mode 100644 types/interactions.d.ts create mode 100644 types/viewport.d.ts diff --git a/src/core.ts b/src/core.ts index c01c0e33..67b8869b 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,28 +1,40 @@ -import deserialize from "./data/deserialize"; -import serialize from "./data/send"; -import discover from "./dom/discover"; -import mutation from "./dom/mutation"; +import { IBindingContainer, IEventBindingPair } from "@clarity-types/core"; +import deserialize from "@src/data/deserialize"; +import discover from "@src/dom/discover"; +import mutation from "@src/dom/mutation"; -window["SERIALIZE"] = serialize; +let bindings: IBindingContainer; window["DESERIALIZE"] = deserialize; /* Initial discovery of DOM */ export function init(): void { + bindings = {}; mutation(); - discover().then(() => { - // DEBUG: Remove later - console.log("done discovery!"); - console.log(window["TRACKER"]["dt"]["duration"] + "ms in " + window["TRACKER"]["dt"]["count"] + " iterations"); - // DEBUG: Serialize DOM - serialize().then((output: string) => { - console.log("Serialized DOM: " + output); - console.log("Serialized DOM Length: " + output.length); - console.log("done serialization!"); - console.log(window["TRACKER"]["st"]["duration"] + "ms in " + window["TRACKER"]["st"]["count"] + " iterations"); - console.log("===================="); - let deserialized = deserialize(output); - console.log("Deserialized DOM: " + deserialized); - console.log("Deserialized DOM Length: " + deserialized.length); - }); + discover(); +} + +export function time(): number { + return performance.now(); +} + +export function bind(target: EventTarget, event: string, listener: EventListener): void { + let eventBindings = bindings[event] || []; + target.addEventListener(event, listener, false); + eventBindings.push({ + target, + listener }); + bindings[event] = eventBindings; +} + +export function teardown(): void { + // Walk through existing list of bindings and remove them all + for (let evt in bindings) { + if (bindings.hasOwnProperty(evt)) { + let eventBindings = bindings[evt] as IEventBindingPair[]; + for (let i = 0; i < eventBindings.length; i++) { + (eventBindings[i].target).removeEventListener(evt, eventBindings[i].listener); + } + } + } } diff --git a/src/data/deserialize.ts b/src/data/deserialize.ts index fd9f428b..c3fad563 100644 --- a/src/data/deserialize.ts +++ b/src/data/deserialize.ts @@ -1,4 +1,4 @@ -import domDeserialize from "../dom/deserialize"; +import domDeserialize from "@src/dom/deserialize"; export default function(payload: string): string { let json = JSON.parse(payload); diff --git a/src/data/send.ts b/src/data/send.ts deleted file mode 100644 index 0626902f..00000000 --- a/src/data/send.ts +++ /dev/null @@ -1,16 +0,0 @@ -import serializeDOM from "../dom/serialize"; -import { Timer } from "../metrics/enums"; -import serializeMetrics from "../metrics/serialize"; -import * as timer from "../metrics/timer"; - -export default async function(): Promise { - let tracker = Timer.Serialize; - timer.start(tracker); - let json = { - dom: await serializeDOM(), - metrics: await serializeMetrics() - }; - let output = JSON.stringify(json); - timer.stop(tracker); - return output; -} diff --git a/src/data/upload.ts b/src/data/upload.ts new file mode 100644 index 00000000..da855727 --- /dev/null +++ b/src/data/upload.ts @@ -0,0 +1,38 @@ +import { Event, Flush, IEvent, Token } from "@clarity-types/data"; +import * as metrics from "@src/metrics/metrics"; +import * as document from "@src/viewport/document"; + +let events: IEvent[] = []; +let wait = 1000; +let timeout: number = null; + +window["PAYLOAD"] = []; + +export function queue(timestamp: number, event: Event, data: Token[], flush: Flush = Flush.Schedule): void { + events.push({ + t: timestamp, + e: event, + d: data + }); + + switch (flush) { + case Flush.Schedule: + clearTimeout(timeout); + timeout = window.setTimeout(send, wait); + break; + case Flush.Force: + clearTimeout(timeout); + send(); + break; + } +} + +function send(): void { + document.compute(); + metrics.compute(); + + let json = JSON.stringify(events); + events = []; + console.log("Json: " + json); + window["PAYLOAD"].push(json); +} diff --git a/src/dom/discover.ts b/src/dom/discover.ts index a5e32c4e..62e6b18e 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,8 +1,18 @@ -import { Timer } from "../metrics/enums"; -import * as timer from "../metrics/timer"; +import { Event, Token } from "@clarity-types/data"; +import { Timer } from "@clarity-types/metrics"; +import { time } from "@src/core"; +import { queue } from "@src/data/upload"; +import serialize from "@src/dom/serialize"; +import * as timer from "@src/metrics/timer"; import processNode from "./node"; -export default async function(): Promise { +export default function(): void { + discover().then((data: Token[]) => { + queue(time(), Event.Discover, data); + }); +} + +async function discover(): Promise { let method = Timer.Discover; timer.start(method); let walker = document.createTreeWalker(document, NodeFilter.SHOW_ALL, null, false); @@ -13,5 +23,7 @@ export default async function(): Promise { node = walker.nextNode(); } console.log("Finished discovering"); + let data = await serialize(method); timer.stop(method); + return data; } diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index e6bcec42..2a34e9e0 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,5 +1,9 @@ -import { Timer } from "../metrics/enums"; -import * as counter from "../metrics/timer"; +import { Event, Token } from "@clarity-types/data"; +import { Timer } from "@clarity-types/metrics"; +import { time } from "@src/core"; +import { queue } from "@src/data/upload"; +import serialize from "@src/dom/serialize"; +import * as timer from "@src/metrics/timer"; import processNode from "./node"; let observer: MutationObserver; @@ -15,47 +19,52 @@ export default function(): void { } function handle(mutations: MutationRecord[]): void { + process(mutations).then((data: Token[]) => { + queue(time(), Event.Mutation, data); + }); +} + +async function process(mutations: MutationRecord[]): Promise { let method = Timer.Mutation; - counter.start(method); + timer.start(method); let length = mutations.length; for (let i = 0; i < length; i++) { - process(mutations[i]); - } - counter.stop(method); -} + let mutation = mutations[i]; + let target = mutation.target; -async function process(mutation: MutationRecord): Promise { - let method = Timer.Mutation; - console.log("Received mutation: " + mutation.type + " | " + mutation.target); - window["MUTATIONS"].push(mutation); + console.log("Received mutation: " + mutation.type + " | " + mutation.target); + window["MUTATIONS"].push(mutation); - let target = mutation.target; - switch (mutation.type) { - case "attributes": - case "characterData": - if (counter.longtasks(method)) { await counter.idle(method); } - processNode(target); - break; - case "childList": - // Process additions - let addedLength = mutation.addedNodes.length; - for (let j = 0; j < addedLength; j++) { - let walker = document.createTreeWalker(mutation.addedNodes[j], NodeFilter.SHOW_ALL, null, false); - let node = walker.currentNode; - while (node) { - if (counter.longtasks(method)) { await counter.idle(method); } - processNode(node); - node = walker.nextNode(); + switch (mutation.type) { + case "attributes": + case "characterData": + if (timer.longtasks(method)) { await timer.idle(method); } + processNode(target); + break; + case "childList": + // Process additions + let addedLength = mutation.addedNodes.length; + for (let j = 0; j < addedLength; j++) { + let walker = document.createTreeWalker(mutation.addedNodes[j], NodeFilter.SHOW_ALL, null, false); + let node = walker.currentNode; + while (node) { + if (timer.longtasks(method)) { await timer.idle(method); } + processNode(node); + node = walker.nextNode(); + } + } + // Process removes + let removedLength = mutation.removedNodes.length; + for (let j = 0; j < removedLength; j++) { + if (timer.longtasks(method)) { await timer.idle(method); } + processNode(mutation.removedNodes[j]); } - } - // Process removes - let removedLength = mutation.removedNodes.length; - for (let j = 0; j < removedLength; j++) { - if (counter.longtasks(method)) { await counter.idle(method); } - processNode(mutation.removedNodes[j]); - } - break; - default: - break; + break; + default: + break; + } } + let data = await serialize(method); + timer.stop(method); + return data; } diff --git a/src/dom/serialize.ts b/src/dom/serialize.ts index a91d8f9d..0da61b68 100644 --- a/src/dom/serialize.ts +++ b/src/dom/serialize.ts @@ -1,22 +1,21 @@ import {Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; -import hash from "../data/hash"; -import {check} from "../data/token"; -import * as counter from "../metrics/counter"; -import { Counter, Timer } from "../metrics/enums"; -import * as timer from "../metrics/timer"; +import { Counter, Timer } from "@clarity-types/metrics"; +import hash from "@src/data/hash"; +import {check} from "@src/data/token"; +import * as counter from "@src/metrics/counter"; +import * as timer from "@src/metrics/timer"; import * as nodes from "./virtualdom"; window["HASH"] = hash; let reference: number = 0; -export default async function(): Promise { - let tracker = Timer.Serialize; +export default async function(method: Timer): Promise { let markup = []; let values = nodes.summarize(); reference = 0; for (let value of values) { - if (timer.longtasks(tracker)) { await timer.idle(tracker); } + if (timer.longtasks(method)) { await timer.idle(method); } let metadata = []; let layouts = []; let data: INodeData = value.data; diff --git a/src/interactions/keyboard.ts b/src/interactions/keyboard.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts new file mode 100644 index 00000000..a9d68046 --- /dev/null +++ b/src/interactions/mouse.ts @@ -0,0 +1,66 @@ +import { Event } from "@clarity-types/data"; +import { IMouseInteraction, Mouse } from "@clarity-types/interactions"; +import { bind, time } from "@src/core"; +import {queue} from "@src/data/upload"; +import { getId } from "@src/dom/virtualdom"; +import serialize from "./serialize"; + +let data: IMouseInteraction[] = []; +let wait = 1000; +let distance = 20; +let timeout: number = null; +let timestamp: number = null; + +export function activate(): void { + bind(document, "mousedown", handler.bind(Mouse.Down)); + bind(document, "mouseup", handler.bind(Mouse.Up)); + bind(document, "mousemove", handler.bind(Mouse.Move)); + bind(document, "mousewheel", handler.bind(Mouse.Wheel)); + bind(document, "click", handler.bind(Mouse.Click)); +} + +function handler(type: Mouse, evt: MouseEvent): void { + let de = document.documentElement; + data.push({ + updated: true, + time: time(), + type, + x: "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null), + y: "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null), + target: evt.target ? getId(evt.target as Node, false) : null, + buttons: evt.buttons + }); + if (timeout) { clearTimeout(timeout); } + timeout = window.setTimeout(schedule, wait); +} + +function schedule(): void { + queue(timestamp, Event.Mouse, serialize(Event.Mouse)); +} + +export function summarize(): IMouseInteraction[] { + let summary: IMouseInteraction[] = []; + let index = 0; + let last = null; + for (let entry of data) { + if (entry.updated) { + let isFirst = index === 0; + if (isFirst + || index === data.length - 1 + || checkDistance(last, entry)) { + timestamp = isFirst ? entry.time : timestamp; + summary.push(entry); + } + index++; + entry.updated = false; + last = entry; + } + } + return summary; +} + +function checkDistance(last: IMouseInteraction, current: IMouseInteraction): boolean { + let dx = last.x - current.x; + let dy = last.y - current.y; + return (dx * dx + dy * dy > distance * distance); +} diff --git a/src/interactions/selection.ts b/src/interactions/selection.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/interactions/serialize.ts b/src/interactions/serialize.ts new file mode 100644 index 00000000..56bd8426 --- /dev/null +++ b/src/interactions/serialize.ts @@ -0,0 +1,25 @@ +import {Event, Token} from "@clarity-types/data"; +import * as mouse from "./mouse"; + +export default function(type: Event): Token[] { + let tokens = []; + + switch (type) { + case Event.Mouse: + let m = mouse.summarize(); + let timestamp: number = null; + for (let i = 0; i < m.length; i++) { + let entry = m[i]; + timestamp = (i === 0) ? entry.time : timestamp; + tokens.push(entry.time - timestamp); + tokens.push(entry.type); + tokens.push(entry.x); + tokens.push(entry.y); + tokens.push(entry.target); + if (entry.buttons > 0) { tokens.push(entry.buttons); } + } + break; + } + + return tokens; +} diff --git a/src/interactions/touch.ts b/src/interactions/touch.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/metrics/counter.ts b/src/metrics/counter.ts index fe76b50d..39957c59 100644 --- a/src/metrics/counter.ts +++ b/src/metrics/counter.ts @@ -1,5 +1,4 @@ -import {ICounter, ICounterSummary} from "@clarity-types/metrics"; -import {Counter} from "./enums"; +import { Counter, ICounter, ICounterSummary} from "@clarity-types/metrics"; let tracker: ICounter = {}; let summary: ICounterSummary = {}; diff --git a/src/metrics/enums.ts b/src/metrics/enums.ts deleted file mode 100644 index 2a3a92ec..00000000 --- a/src/metrics/enums.ts +++ /dev/null @@ -1,31 +0,0 @@ -export enum Timer { - Discover = "dt", - Mutation = "mt", - Layout = "lt", - Serialize = "st", - Wireup = "wt", - Active = "at" -} - -export enum Counter { - Nodes = "nc", - Bytes = "bc", - Mutations = "mc", - Swipes = "sc", -} - -export enum Histogram { - PointerDistance = "ph", - ViewportX = "xh", - ViewportY = "yh", - ViewportWidth = "wh", - ViewportHeight = "hh", - DocumentWidth = "dwh", - DocumentHeight = "dhh" -} - -export enum Mark { - Click = "cm", - Error = "em", - Interaction = "ic" -} diff --git a/src/metrics/histogram.ts b/src/metrics/histogram.ts index 31a4b701..5867ea8d 100644 --- a/src/metrics/histogram.ts +++ b/src/metrics/histogram.ts @@ -1,5 +1,4 @@ -import {IHistogram, IHistogramSummary} from "@clarity-types/metrics"; -import {Histogram} from "./enums"; +import { Histogram, IHistogram, IHistogramSummary} from "@clarity-types/metrics"; let tracker: IHistogram = {}; let summary: IHistogramSummary = {}; diff --git a/src/metrics/mark.ts b/src/metrics/mark.ts index 3ac1d8f0..39bfffe5 100644 --- a/src/metrics/mark.ts +++ b/src/metrics/mark.ts @@ -1,5 +1,4 @@ -import {IMark, IMarkSummary} from "@clarity-types/metrics"; -import {Mark} from "./enums"; +import {IMark, IMarkSummary, Mark } from "@clarity-types/metrics"; let tracker: IMark[] = []; let summary: IMarkSummary[] = []; diff --git a/src/metrics/metrics.ts b/src/metrics/metrics.ts new file mode 100644 index 00000000..44708b03 --- /dev/null +++ b/src/metrics/metrics.ts @@ -0,0 +1,8 @@ +import { Event, Flush } from "@clarity-types/data"; +import { time } from "@src/core"; +import { queue } from "@src/data/upload"; +import serialize from "./serialize"; + +export function compute(): void { + queue(time(), Event.Metrics, serialize(), Flush.None); +} diff --git a/src/metrics/serialize.ts b/src/metrics/serialize.ts index 457db00b..00a11d76 100644 --- a/src/metrics/serialize.ts +++ b/src/metrics/serialize.ts @@ -1,14 +1,11 @@ import {Token} from "@clarity-types/data"; import * as counter from "./counter"; -import { Timer } from "./enums"; import * as histogram from "./histogram"; import * as mark from "./mark"; import * as timer from "./timer"; -export default async function(): Promise { - let tracker = Timer.Serialize; +export default function(): Token[] { let metrics = []; - if (timer.longtasks(tracker)) { await timer.idle(tracker); } // Serialize counters let counters = counter.summarize(); diff --git a/src/metrics/timer.ts b/src/metrics/timer.ts index 884b7d96..33cb5e2f 100644 --- a/src/metrics/timer.ts +++ b/src/metrics/timer.ts @@ -1,5 +1,4 @@ -import {ITimer, ITimerSummary} from "@clarity-types/metrics"; -import {Timer} from "./enums"; +import {ITimer, ITimerSummary, Timer } from "@clarity-types/metrics"; let tracker: ITimer = {}; let summary: ITimerSummary = {}; diff --git a/src/performance/navigation.ts b/src/performance/navigation.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/performance/resource.ts b/src/performance/resource.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/viewport/document.ts b/src/viewport/document.ts new file mode 100644 index 00000000..af2f6243 --- /dev/null +++ b/src/viewport/document.ts @@ -0,0 +1,40 @@ +import { Event, Flush } from "@clarity-types/data"; +import { IDocumentSize } from "@clarity-types/viewport"; +import { time } from "@src/core"; +import {queue} from "@src/data/upload"; +import serialize from "./serialize"; + +let data: IDocumentSize; + +export function activate(): void { + recompute(); +} + +function recompute(): void { + let body = document.body; + let doc = document.documentElement; + let bodyClientHeight = body ? body.clientHeight : null; + let bodyScrollHeight = body ? body.scrollHeight : null; + let bodyOffsetHeight = body ? body.offsetHeight : null; + let documentClientHeight = doc ? doc.clientHeight : null; + let documentScrollHeight = doc ? doc.scrollHeight : null; + let documentOffsetHeight = doc ? doc.offsetHeight : null; + let documentHeight = Math.max(bodyClientHeight, bodyScrollHeight, bodyOffsetHeight, + documentClientHeight, documentScrollHeight, documentOffsetHeight); + + data = { + width: body ? body.clientWidth : null, + height: documentHeight, + updated: true + }; + + queue(time(), Event.Document, serialize(Event.Document), Flush.None); +} + +export function compute(): void { + recompute(); +} + +export function summarize(): IDocumentSize { + return data.updated ? data : null; +} diff --git a/src/viewport/resize.ts b/src/viewport/resize.ts new file mode 100644 index 00000000..35fb5a0b --- /dev/null +++ b/src/viewport/resize.ts @@ -0,0 +1,25 @@ +import { Event } from "@clarity-types/data"; +import { IResizeViewport } from "@clarity-types/viewport"; +import { bind, time } from "@src/core"; +import {queue} from "@src/data/upload"; +import serialize from "./serialize"; + +let data: IResizeViewport; + +export function activate(): void { + bind(window, "resize", recompute); + recompute(); +} + +function recompute(): void { + data = { + width: "innerWidth" in window ? window.innerWidth : document.documentElement.clientWidth, + height: "innerHeight" in window ? window.innerHeight : document.documentElement.clientHeight, + updated: true + }; + queue(time(), Event.Resize, serialize(Event.Resize)); +} + +export function summarize(): IResizeViewport { + return data.updated ? data : null; +} diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts new file mode 100644 index 00000000..14758af8 --- /dev/null +++ b/src/viewport/scroll.ts @@ -0,0 +1,55 @@ +import { Event } from "@clarity-types/data"; +import { IScrollViewport } from "@clarity-types/viewport"; +import { bind, time } from "@src/core"; +import {queue} from "@src/data/upload"; +import serialize from "./serialize"; + +let data: IScrollViewport[] = []; +let wait = 1000; +let distance = 20; +let timeout: number = null; +let timestamp: number = null; + +export function activate(): void { + bind(window, "scroll", recompute); + recompute(); +} + +function recompute(): void { + let x = "pageXOffset" in window ? window.pageXOffset : document.documentElement.scrollLeft; + let y = "pageYOffset" in window ? window.pageYOffset : document.documentElement.scrollTop; + data.push({time: time(), x, y, updated: true}); + if (timeout) { clearTimeout(timeout); } + timeout = window.setTimeout(schedule, wait); +} + +function schedule(): void { + queue(timestamp, Event.Scroll, serialize(Event.Scroll)); +} + +export function summarize(): IScrollViewport[] { + let summary: IScrollViewport[] = []; + let index = 0; + let last = null; + for (let entry of data) { + if (entry.updated) { + let isFirst = index === 0; + if (isFirst + || index === data.length - 1 + || checkDistance(last, entry)) { + timestamp = isFirst ? entry.time : timestamp; + summary.push(entry); + } + index++; + entry.updated = false; + last = entry; + } + } + return summary; +} + +function checkDistance(last: IScrollViewport, current: IScrollViewport): boolean { + let dx = last.x - current.x; + let dy = last.y - current.y; + return (dx * dx + dy * dy > distance * distance); +} diff --git a/src/viewport/serialize.ts b/src/viewport/serialize.ts new file mode 100644 index 00000000..e86b7831 --- /dev/null +++ b/src/viewport/serialize.ts @@ -0,0 +1,33 @@ +import {Event, Token} from "@clarity-types/data"; +import * as document from "./document"; +import * as resize from "./resize"; +import * as scroll from "./scroll"; + +export default function(type: Event): Token[] { + let tokens = []; + + switch (type) { + case Event.Resize: + let r = resize.summarize(); + tokens.push(r.width); + tokens.push(r.height); + break; + case Event.Document: + let d = document.summarize(); + tokens.push(d.width); + tokens.push(d.height); + case Event.Scroll: + let s = scroll.summarize(); + let timestamp: number = null; + for (let i = 0; i < s.length; i++) { + let entry = s[i]; + timestamp = (i === 0) ? entry.time : timestamp; + tokens.push(entry.time - timestamp); + tokens.push(entry.x); + tokens.push(entry.y); + } + break; + } + + return tokens; +} diff --git a/src/viewport/visibility.ts b/src/viewport/visibility.ts new file mode 100644 index 00000000..a0162f44 --- /dev/null +++ b/src/viewport/visibility.ts @@ -0,0 +1,26 @@ +import { Event } from "@clarity-types/data"; +import { IPageVisibility } from "@clarity-types/viewport"; +import { bind, time } from "@src/core"; +import {queue} from "@src/data/upload"; +import serialize from "./serialize"; + +let data: IPageVisibility; + +export function activate(): void { + bind(window, "pagehide", recompute); + bind(window, "pageshow", recompute); + bind(document, "visibilitychange", recompute); + recompute(); +} + +function recompute(): void { + data = { + visible: "visibilityState" in document ? document.visibilityState : "default", + updated: true + }; + queue(time(), Event.Visibility, serialize(Event.Visibility)); +} + +export function summarize(): IPageVisibility { + return data.updated ? data : null; +} diff --git a/types/core.d.ts b/types/core.d.ts index 2426230b..ed97441c 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -1,2 +1,8 @@ -export const enum State { +export interface IEventBindingPair { + target: EventTarget; + listener: EventListener; +} + +export interface IBindingContainer { + [key: string]: IEventBindingPair[]; } diff --git a/types/data.d.ts b/types/data.d.ts index 3bfad016..962443ba 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1 +1,29 @@ export type Token = (string | number | number[] | string[]); + +export const enum Event { + Discover, + Mutation, + Metrics, + Mouse, + Touch, + Keyboard, + Selection, + Resize, + Scroll, + Document, + Visibility, + Network, + Performance +} + +export const enum Flush { + Schedule, + Force, + None +} + +export interface IEvent { + t: number; + e: Event; + d: Token[]; +} diff --git a/types/interactions.d.ts b/types/interactions.d.ts new file mode 100644 index 00000000..74158099 --- /dev/null +++ b/types/interactions.d.ts @@ -0,0 +1,17 @@ +export const enum Mouse { + Down = "d", + Up = "u", + Move = "m", + Wheel = "w", + Click = "c" +} + +interface IMouseInteraction { + updated: boolean; + time: number; + type: Mouse; + x: number; + y: number; + target: number; + buttons: number; +} diff --git a/types/metrics.d.ts b/types/metrics.d.ts index 689794d1..d1f32280 100644 --- a/types/metrics.d.ts +++ b/types/metrics.d.ts @@ -1,3 +1,34 @@ +// Metric Names +export const enum Timer { + Discover = "dt", + Mutation = "mt", + Wireup = "wt", + Active = "at" +} + +export const enum Counter { + Nodes = "nc", + Bytes = "bc", + Mutations = "mc", + Swipes = "sc", +} + +export const enum Histogram { + PointerDistance = "ph", + ViewportX = "xh", + ViewportY = "yh", + ViewportWidth = "wh", + ViewportHeight = "hh", + DocumentWidth = "dwh", + DocumentHeight = "dhh" +} + +export const enum Mark { + Click = "cm", + Error = "em", + Interaction = "ic" +} + // Counter export interface ICounter { [key: number]: ICounterValue; diff --git a/types/viewport.d.ts b/types/viewport.d.ts new file mode 100644 index 00000000..8b0aca19 --- /dev/null +++ b/types/viewport.d.ts @@ -0,0 +1,23 @@ +export interface IResizeViewport { + width: number; + height: number; + updated: boolean; +} + +export interface IScrollViewport { + time: number; + x: number; + y: number; + updated: boolean; +} + +export interface IDocumentSize { + width: number; + height: number; + updated: boolean; +} + +export interface IPageVisibility { + visible: string; + updated: boolean; +} From 52850d503e55ecb9749f1e566a0a6589cdf69243 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 16 Jul 2019 06:08:43 -0700 Subject: [PATCH 015/105] Initialize various modules on clarity startup --- src/clarity.ts | 58 +++++++++++++++++++++++++++++-- src/core.ts | 40 --------------------- src/dom/discover.ts | 4 +-- src/dom/mutation.ts | 9 +++-- src/interactions/mouse.ts | 14 ++++---- src/metrics/metrics.ts | 2 +- src/viewport/document.ts | 4 +-- src/viewport/resize.ts | 4 +-- src/viewport/scroll.ts | 4 +-- src/viewport/visibility.ts | 4 +-- types/{core.d.ts => clarity.d.ts} | 0 11 files changed, 81 insertions(+), 62 deletions(-) delete mode 100644 src/core.ts rename types/{core.d.ts => clarity.d.ts} (100%) diff --git a/src/clarity.ts b/src/clarity.ts index 2ede457d..fd5aa1c8 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,7 +1,61 @@ -import {init} from "./core"; +import { IBindingContainer, IEventBindingPair } from "@clarity-types/clarity"; +import deserialize from "@src/data/deserialize"; +import * as discover from "@src/dom/discover"; +import * as mutation from "@src/dom/mutation"; +import * as mouse from "@src/interactions/mouse"; +import * as document from "@src/viewport/document"; +import * as resize from "@src/viewport/resize"; +import * as scroll from "@src/viewport/scroll"; +import * as visibility from "@src/viewport/visibility"; +let bindings: IBindingContainer; +window["DESERIALIZE"] = deserialize; + +/* Initial discovery of DOM */ export function start(): void { - init(); + bindings = {}; + + // DOM + mutation.start(); + discover.start(); + + // Viewport + document.start(); + resize.start(); + visibility.start(); + scroll.start(); + + // Pointer + mouse.start(); + +} + +export function time(): number { + return Math.round(performance.now()); +} + +export function bind(target: EventTarget, event: string, listener: EventListener): void { + let eventBindings = bindings[event] || []; + target.addEventListener(event, listener, false); + eventBindings.push({ + target, + listener + }); + bindings[event] = eventBindings; +} + +export function end(): void { + // Walk through existing list of bindings and remove them all + for (let evt in bindings) { + if (bindings.hasOwnProperty(evt)) { + let eventBindings = bindings[evt] as IEventBindingPair[]; + for (let i = 0; i < eventBindings.length; i++) { + (eventBindings[i].target).removeEventListener(evt, eventBindings[i].listener); + } + } + } + + mutation.end(); } start(); diff --git a/src/core.ts b/src/core.ts deleted file mode 100644 index 67b8869b..00000000 --- a/src/core.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { IBindingContainer, IEventBindingPair } from "@clarity-types/core"; -import deserialize from "@src/data/deserialize"; -import discover from "@src/dom/discover"; -import mutation from "@src/dom/mutation"; - -let bindings: IBindingContainer; -window["DESERIALIZE"] = deserialize; - -/* Initial discovery of DOM */ -export function init(): void { - bindings = {}; - mutation(); - discover(); -} - -export function time(): number { - return performance.now(); -} - -export function bind(target: EventTarget, event: string, listener: EventListener): void { - let eventBindings = bindings[event] || []; - target.addEventListener(event, listener, false); - eventBindings.push({ - target, - listener - }); - bindings[event] = eventBindings; -} - -export function teardown(): void { - // Walk through existing list of bindings and remove them all - for (let evt in bindings) { - if (bindings.hasOwnProperty(evt)) { - let eventBindings = bindings[evt] as IEventBindingPair[]; - for (let i = 0; i < eventBindings.length; i++) { - (eventBindings[i].target).removeEventListener(evt, eventBindings[i].listener); - } - } - } -} diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 62e6b18e..d3ff6c89 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,12 +1,12 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; -import { time } from "@src/core"; +import { time } from "@src/clarity"; import { queue } from "@src/data/upload"; import serialize from "@src/dom/serialize"; import * as timer from "@src/metrics/timer"; import processNode from "./node"; -export default function(): void { +export function start(): void { discover().then((data: Token[]) => { queue(time(), Event.Discover, data); }); diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 2a34e9e0..27aa99a4 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,6 +1,6 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; -import { time } from "@src/core"; +import { time } from "@src/clarity"; import { queue } from "@src/data/upload"; import serialize from "@src/dom/serialize"; import * as timer from "@src/metrics/timer"; @@ -9,7 +9,7 @@ import processNode from "./node"; let observer: MutationObserver; window["MUTATIONS"] = []; -export default function(): void { +export function start(): void { console.log("Listening for mutations..."); if (observer) { observer.disconnect(); @@ -18,6 +18,11 @@ export default function(): void { observer.observe(document, { attributes: true, childList: true, characterData: true, subtree: true }); } +export function end(): void { + observer.disconnect(); + observer = null; +} + function handle(mutations: MutationRecord[]): void { process(mutations).then((data: Token[]) => { queue(time(), Event.Mutation, data); diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index a9d68046..8ce6a1b3 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -1,6 +1,6 @@ import { Event } from "@clarity-types/data"; import { IMouseInteraction, Mouse } from "@clarity-types/interactions"; -import { bind, time } from "@src/core"; +import { bind, time } from "@src/clarity"; import {queue} from "@src/data/upload"; import { getId } from "@src/dom/virtualdom"; import serialize from "./serialize"; @@ -11,12 +11,12 @@ let distance = 20; let timeout: number = null; let timestamp: number = null; -export function activate(): void { - bind(document, "mousedown", handler.bind(Mouse.Down)); - bind(document, "mouseup", handler.bind(Mouse.Up)); - bind(document, "mousemove", handler.bind(Mouse.Move)); - bind(document, "mousewheel", handler.bind(Mouse.Wheel)); - bind(document, "click", handler.bind(Mouse.Click)); +export function start(): void { + bind(document, "mousedown", handler.bind(this, Mouse.Down)); + bind(document, "mouseup", handler.bind(this, Mouse.Up)); + bind(document, "mousemove", handler.bind(this, Mouse.Move)); + bind(document, "mousewheel", handler.bind(this, Mouse.Wheel)); + bind(document, "click", handler.bind(this, Mouse.Click)); } function handler(type: Mouse, evt: MouseEvent): void { diff --git a/src/metrics/metrics.ts b/src/metrics/metrics.ts index 44708b03..3f55d71a 100644 --- a/src/metrics/metrics.ts +++ b/src/metrics/metrics.ts @@ -1,5 +1,5 @@ import { Event, Flush } from "@clarity-types/data"; -import { time } from "@src/core"; +import { time } from "@src/clarity"; import { queue } from "@src/data/upload"; import serialize from "./serialize"; diff --git a/src/viewport/document.ts b/src/viewport/document.ts index af2f6243..31d867a9 100644 --- a/src/viewport/document.ts +++ b/src/viewport/document.ts @@ -1,12 +1,12 @@ import { Event, Flush } from "@clarity-types/data"; import { IDocumentSize } from "@clarity-types/viewport"; -import { time } from "@src/core"; +import { time } from "@src/clarity"; import {queue} from "@src/data/upload"; import serialize from "./serialize"; let data: IDocumentSize; -export function activate(): void { +export function start(): void { recompute(); } diff --git a/src/viewport/resize.ts b/src/viewport/resize.ts index 35fb5a0b..2a1924fc 100644 --- a/src/viewport/resize.ts +++ b/src/viewport/resize.ts @@ -1,12 +1,12 @@ import { Event } from "@clarity-types/data"; import { IResizeViewport } from "@clarity-types/viewport"; -import { bind, time } from "@src/core"; +import { bind, time } from "@src/clarity"; import {queue} from "@src/data/upload"; import serialize from "./serialize"; let data: IResizeViewport; -export function activate(): void { +export function start(): void { bind(window, "resize", recompute); recompute(); } diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts index 14758af8..72530632 100644 --- a/src/viewport/scroll.ts +++ b/src/viewport/scroll.ts @@ -1,6 +1,6 @@ import { Event } from "@clarity-types/data"; import { IScrollViewport } from "@clarity-types/viewport"; -import { bind, time } from "@src/core"; +import { bind, time } from "@src/clarity"; import {queue} from "@src/data/upload"; import serialize from "./serialize"; @@ -10,7 +10,7 @@ let distance = 20; let timeout: number = null; let timestamp: number = null; -export function activate(): void { +export function start(): void { bind(window, "scroll", recompute); recompute(); } diff --git a/src/viewport/visibility.ts b/src/viewport/visibility.ts index a0162f44..b85aba00 100644 --- a/src/viewport/visibility.ts +++ b/src/viewport/visibility.ts @@ -1,12 +1,12 @@ import { Event } from "@clarity-types/data"; import { IPageVisibility } from "@clarity-types/viewport"; -import { bind, time } from "@src/core"; +import { bind, time } from "@src/clarity"; import {queue} from "@src/data/upload"; import serialize from "./serialize"; let data: IPageVisibility; -export function activate(): void { +export function start(): void { bind(window, "pagehide", recompute); bind(window, "pageshow", recompute); bind(document, "visibilitychange", recompute); diff --git a/types/core.d.ts b/types/clarity.d.ts similarity index 100% rename from types/core.d.ts rename to types/clarity.d.ts From 66fb64fd154e803ca7d5b99bad4c7897a98d908e Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 30 Jul 2019 06:54:54 -0700 Subject: [PATCH 016/105] Refactoring code to restructure core methods --- src/clarity.ts | 30 +++--------------------------- src/core/config.ts | 0 src/core/event.ts | 26 ++++++++++++++++++++++++++ src/core/queue.ts | 38 ++++++++++++++++++++++++++++++++++++++ src/core/recompute.ts | 7 +++++++ src/core/time.ts | 3 +++ src/data/upload.ts | 35 ++--------------------------------- src/dom/discover.ts | 4 ++-- src/dom/mutation.ts | 4 ++-- src/interactions/mouse.ts | 5 +++-- src/metrics/metrics.ts | 4 ++-- src/viewport/document.ts | 4 ++-- src/viewport/resize.ts | 5 +++-- src/viewport/scroll.ts | 5 +++-- src/viewport/visibility.ts | 5 +++-- 15 files changed, 99 insertions(+), 76 deletions(-) create mode 100644 src/core/config.ts create mode 100644 src/core/event.ts create mode 100644 src/core/queue.ts create mode 100644 src/core/recompute.ts create mode 100644 src/core/time.ts diff --git a/src/clarity.ts b/src/clarity.ts index fd5aa1c8..e65b8e96 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,4 +1,4 @@ -import { IBindingContainer, IEventBindingPair } from "@clarity-types/clarity"; +import * as event from "@src/core/event"; import deserialize from "@src/data/deserialize"; import * as discover from "@src/dom/discover"; import * as mutation from "@src/dom/mutation"; @@ -8,12 +8,11 @@ import * as resize from "@src/viewport/resize"; import * as scroll from "@src/viewport/scroll"; import * as visibility from "@src/viewport/visibility"; -let bindings: IBindingContainer; window["DESERIALIZE"] = deserialize; /* Initial discovery of DOM */ export function start(): void { - bindings = {}; + event.reset(); // DOM mutation.start(); @@ -30,31 +29,8 @@ export function start(): void { } -export function time(): number { - return Math.round(performance.now()); -} - -export function bind(target: EventTarget, event: string, listener: EventListener): void { - let eventBindings = bindings[event] || []; - target.addEventListener(event, listener, false); - eventBindings.push({ - target, - listener - }); - bindings[event] = eventBindings; -} - export function end(): void { - // Walk through existing list of bindings and remove them all - for (let evt in bindings) { - if (bindings.hasOwnProperty(evt)) { - let eventBindings = bindings[evt] as IEventBindingPair[]; - for (let i = 0; i < eventBindings.length; i++) { - (eventBindings[i].target).removeEventListener(evt, eventBindings[i].listener); - } - } - } - + event.reset(); mutation.end(); } diff --git a/src/core/config.ts b/src/core/config.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/core/event.ts b/src/core/event.ts new file mode 100644 index 00000000..bef3efe8 --- /dev/null +++ b/src/core/event.ts @@ -0,0 +1,26 @@ +import { IBindingContainer, IEventBindingPair } from "@clarity-types/clarity"; + +let bindings: IBindingContainer = {}; + +export function bind(target: EventTarget, event: string, listener: EventListener): void { + let eventBindings = bindings[event] || []; + target.addEventListener(event, listener, false); + eventBindings.push({ + target, + listener + }); + bindings[event] = eventBindings; +} + +export function reset(): void { + // Walk through existing list of bindings and remove them all + for (let evt in bindings) { + if (bindings.hasOwnProperty(evt)) { + let eventBindings = bindings[evt] as IEventBindingPair[]; + for (let i = 0; i < eventBindings.length; i++) { + (eventBindings[i].target).removeEventListener(evt, eventBindings[i].listener); + } + } + } + bindings = {}; +} diff --git a/src/core/queue.ts b/src/core/queue.ts new file mode 100644 index 00000000..ad704f94 --- /dev/null +++ b/src/core/queue.ts @@ -0,0 +1,38 @@ +import { Event, Flush, IEvent, Token } from "@clarity-types/data"; +import upload from "@src/data/upload"; +import recompute from "./recompute"; + +let events: IEvent[] = []; +let wait = 1000; +let timeout: number = null; + +window["PAYLOAD"] = []; + +export default function(timestamp: number, event: Event, data: Token[], flush: Flush = Flush.Schedule): void { + events.push({ + t: timestamp, + e: event, + d: data + }); + + switch (flush) { + case Flush.Schedule: + clearTimeout(timeout); + timeout = window.setTimeout(dequeue, wait); + break; + case Flush.Force: + clearTimeout(timeout); + dequeue(); + break; + } +} + +function dequeue(): void { + recompute(); + upload(events); + reset(); +} + +function reset(): void { + events = []; +} diff --git a/src/core/recompute.ts b/src/core/recompute.ts new file mode 100644 index 00000000..1377e8a9 --- /dev/null +++ b/src/core/recompute.ts @@ -0,0 +1,7 @@ +import * as metrics from "@src/metrics/metrics"; +import * as document from "@src/viewport/document"; + +export default function(): void { + document.compute(); + metrics.compute(); +} diff --git a/src/core/time.ts b/src/core/time.ts new file mode 100644 index 00000000..38e48f6e --- /dev/null +++ b/src/core/time.ts @@ -0,0 +1,3 @@ +export default function(): number { + return Math.round(performance.now()); +} diff --git a/src/data/upload.ts b/src/data/upload.ts index da855727..2064873c 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,38 +1,7 @@ -import { Event, Flush, IEvent, Token } from "@clarity-types/data"; -import * as metrics from "@src/metrics/metrics"; -import * as document from "@src/viewport/document"; - -let events: IEvent[] = []; -let wait = 1000; -let timeout: number = null; - -window["PAYLOAD"] = []; - -export function queue(timestamp: number, event: Event, data: Token[], flush: Flush = Flush.Schedule): void { - events.push({ - t: timestamp, - e: event, - d: data - }); - - switch (flush) { - case Flush.Schedule: - clearTimeout(timeout); - timeout = window.setTimeout(send, wait); - break; - case Flush.Force: - clearTimeout(timeout); - send(); - break; - } -} - -function send(): void { - document.compute(); - metrics.compute(); +import { IEvent } from "@clarity-types/data"; +export default function(events: IEvent[]): void { let json = JSON.stringify(events); - events = []; console.log("Json: " + json); window["PAYLOAD"].push(json); } diff --git a/src/dom/discover.ts b/src/dom/discover.ts index d3ff6c89..20cb493f 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,7 +1,7 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; -import { time } from "@src/clarity"; -import { queue } from "@src/data/upload"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import serialize from "@src/dom/serialize"; import * as timer from "@src/metrics/timer"; import processNode from "./node"; diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 27aa99a4..e25d8ea3 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,7 +1,7 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; -import { time } from "@src/clarity"; -import { queue } from "@src/data/upload"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import serialize from "@src/dom/serialize"; import * as timer from "@src/metrics/timer"; import processNode from "./node"; diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index 8ce6a1b3..50a06bda 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -1,7 +1,8 @@ import { Event } from "@clarity-types/data"; import { IMouseInteraction, Mouse } from "@clarity-types/interactions"; -import { bind, time } from "@src/clarity"; -import {queue} from "@src/data/upload"; +import { bind } from "@src/core/event"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import { getId } from "@src/dom/virtualdom"; import serialize from "./serialize"; diff --git a/src/metrics/metrics.ts b/src/metrics/metrics.ts index 3f55d71a..afba8a12 100644 --- a/src/metrics/metrics.ts +++ b/src/metrics/metrics.ts @@ -1,6 +1,6 @@ import { Event, Flush } from "@clarity-types/data"; -import { time } from "@src/clarity"; -import { queue } from "@src/data/upload"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import serialize from "./serialize"; export function compute(): void { diff --git a/src/viewport/document.ts b/src/viewport/document.ts index 31d867a9..bc788bac 100644 --- a/src/viewport/document.ts +++ b/src/viewport/document.ts @@ -1,7 +1,7 @@ import { Event, Flush } from "@clarity-types/data"; import { IDocumentSize } from "@clarity-types/viewport"; -import { time } from "@src/clarity"; -import {queue} from "@src/data/upload"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import serialize from "./serialize"; let data: IDocumentSize; diff --git a/src/viewport/resize.ts b/src/viewport/resize.ts index 2a1924fc..10012fff 100644 --- a/src/viewport/resize.ts +++ b/src/viewport/resize.ts @@ -1,7 +1,8 @@ import { Event } from "@clarity-types/data"; import { IResizeViewport } from "@clarity-types/viewport"; -import { bind, time } from "@src/clarity"; -import {queue} from "@src/data/upload"; +import { bind } from "@src/core/event"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import serialize from "./serialize"; let data: IResizeViewport; diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts index 72530632..7d8bfbe6 100644 --- a/src/viewport/scroll.ts +++ b/src/viewport/scroll.ts @@ -1,7 +1,8 @@ import { Event } from "@clarity-types/data"; import { IScrollViewport } from "@clarity-types/viewport"; -import { bind, time } from "@src/clarity"; -import {queue} from "@src/data/upload"; +import { bind } from "@src/core/event"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import serialize from "./serialize"; let data: IScrollViewport[] = []; diff --git a/src/viewport/visibility.ts b/src/viewport/visibility.ts index b85aba00..2dd9255d 100644 --- a/src/viewport/visibility.ts +++ b/src/viewport/visibility.ts @@ -1,7 +1,8 @@ import { Event } from "@clarity-types/data"; import { IPageVisibility } from "@clarity-types/viewport"; -import { bind, time } from "@src/clarity"; -import {queue} from "@src/data/upload"; +import { bind } from "@src/core/event"; +import queue from "@src/core/queue"; +import time from "@src/core/time"; import serialize from "./serialize"; let data: IPageVisibility; From b4fdc606739db2a63e95b15896750e92c7b232ca Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 1 Aug 2019 08:48:05 -0700 Subject: [PATCH 017/105] Adding early support for config --- src/core/config.ts | 10 ++++++++++ src/core/event.ts | 2 +- src/core/queue.ts | 6 +----- src/interactions/mouse.ts | 5 +++-- types/clarity.d.ts | 10 +++------- types/core.d.ts | 21 +++++++++++++++++++++ 6 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 types/core.d.ts diff --git a/src/core/config.ts b/src/core/config.ts index e69de29b..f1da18ad 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -0,0 +1,10 @@ +import { IConfig } from "@clarity-types/core"; + +export let config: IConfig = { + /* Interactions */ + lookahead: 250, + distance: 20, + /* Data */ + delay: 1000, + upload: null, +}; diff --git a/src/core/event.ts b/src/core/event.ts index bef3efe8..c6638975 100644 --- a/src/core/event.ts +++ b/src/core/event.ts @@ -1,4 +1,4 @@ -import { IBindingContainer, IEventBindingPair } from "@clarity-types/clarity"; +import { IBindingContainer, IEventBindingPair } from "@clarity-types/core"; let bindings: IBindingContainer = {}; diff --git a/src/core/queue.ts b/src/core/queue.ts index ad704f94..88dd1812 100644 --- a/src/core/queue.ts +++ b/src/core/queue.ts @@ -9,11 +9,7 @@ let timeout: number = null; window["PAYLOAD"] = []; export default function(timestamp: number, event: Event, data: Token[], flush: Flush = Flush.Schedule): void { - events.push({ - t: timestamp, - e: event, - d: data - }); + events.push({t: timestamp, e: event, d: data}); switch (flush) { case Flush.Schedule: diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index 50a06bda..b2f7b8de 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -1,5 +1,6 @@ import { Event } from "@clarity-types/data"; import { IMouseInteraction, Mouse } from "@clarity-types/interactions"; +import { config } from "@src/core/config"; import { bind } from "@src/core/event"; import queue from "@src/core/queue"; import time from "@src/core/time"; @@ -7,8 +8,8 @@ import { getId } from "@src/dom/virtualdom"; import serialize from "./serialize"; let data: IMouseInteraction[] = []; -let wait = 1000; -let distance = 20; +let wait = config.lookahead; +let distance = config.distance; let timeout: number = null; let timestamp: number = null; diff --git a/types/clarity.d.ts b/types/clarity.d.ts index ed97441c..a84477ed 100644 --- a/types/clarity.d.ts +++ b/types/clarity.d.ts @@ -1,8 +1,4 @@ -export interface IEventBindingPair { - target: EventTarget; - listener: EventListener; -} - -export interface IBindingContainer { - [key: string]: IEventBindingPair[]; +export interface IClarity { + start: () => void; + end: () => void; } diff --git a/types/core.d.ts b/types/core.d.ts new file mode 100644 index 00000000..99f76345 --- /dev/null +++ b/types/core.d.ts @@ -0,0 +1,21 @@ +export interface IEventBindingPair { + target: EventTarget; + listener: EventListener; +} + +export interface IBindingContainer { + [key: string]: IEventBindingPair[]; +} + +export interface IConfig { + /* Interactions */ + // Each interaction is going to wait until the specified milliseconds below before marking the end of interaction + lookahead: number; + distance: number; + /* Data */ + // Each new event is going to delay data upload to server by this number of milliseconds + delay: number; + // Pointer to the function which would be responsible for sending the data + // If left unspecified, raw payloads will be uploaded to the upload url endpoint + upload: () => void; +} From 6675bbb217816ec22db0eb72152401a9ac98d799 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 2 Aug 2019 08:57:38 -0700 Subject: [PATCH 018/105] Isolating code to decode in a separate folder --- decode/clarity.ts | 102 +++++++++++++ decode/dom.ts | 89 +++++++++++ src/clarity.ts | 3 - src/data/deserialize.ts | 6 - src/dom/deserialize.ts | 148 ------------------- src/dom/discover.ts | 4 +- src/dom/{serialize.ts => encode.ts} | 0 src/dom/mutation.ts | 4 +- src/interactions/{serialize.ts => encode.ts} | 0 src/interactions/mouse.ts | 4 +- src/metrics/{serialize.ts => encode.ts} | 0 src/metrics/metrics.ts | 4 +- src/viewport/document.ts | 4 +- src/viewport/{serialize.ts => encode.ts} | 0 src/viewport/resize.ts | 4 +- src/viewport/scroll.ts | 4 +- src/viewport/visibility.ts | 4 +- tsconfig.json | 2 + types/data.d.ts | 8 +- types/dom.d.ts | 10 ++ 20 files changed, 226 insertions(+), 174 deletions(-) create mode 100644 decode/clarity.ts create mode 100644 decode/dom.ts delete mode 100644 src/data/deserialize.ts delete mode 100644 src/dom/deserialize.ts rename src/dom/{serialize.ts => encode.ts} (100%) rename src/interactions/{serialize.ts => encode.ts} (100%) rename src/metrics/{serialize.ts => encode.ts} (100%) rename src/viewport/{serialize.ts => encode.ts} (100%) diff --git a/decode/clarity.ts b/decode/clarity.ts new file mode 100644 index 00000000..db353c3c --- /dev/null +++ b/decode/clarity.ts @@ -0,0 +1,102 @@ +import { Event, IDecodedEvent, IEvent, Token } from "@clarity-types/data"; +import { IDecodedNode } from "@clarity-types/dom"; +import dom from "@decode/dom"; + +let nodes = {}; +let placeholder = document.createElement("iframe"); + +export function json(payload: string): IDecodedEvent[] { + let decoded: IDecodedEvent[] = []; + let encoded: IEvent[] = JSON.parse(payload); + for (let entry of encoded) { + let exploded = { time: entry.t, event: entry.e, data: null as Token[] }; + switch (entry.e) { + case Event.Discover: + case Event.Mutation: + exploded.data = dom(entry.d); + break; + } + decoded.push(exploded); + } + return decoded; +} + +export function html(payload: string): string { + let decoded = json(payload); + for (let entry of decoded) { + switch (entry.event) { + case Event.Discover: + case Event.Mutation: + markup(entry.data); + break; + } + } + return placeholder.contentDocument.documentElement.outerHTML; +} + +function markup(data: IDecodedNode[]): void { + let doc = placeholder.contentDocument; + for (let node of data) { + let parent = element(node.parent); + let next = element(node.next); + switch (node.tag) { + case "*D": + if (typeof XMLSerializer !== "undefined" && false) { + doc.open(); + doc.write(new XMLSerializer().serializeToString( + doc.implementation.createDocumentType( + node.attributes["name"], + node.attributes["publicId"], + node.attributes["systemId"] + ) + )); + doc.close(); + } + break; + case "*T": + let textElement = element(node.id); + textElement = textElement ? textElement : doc.createTextNode(null); + textElement.nodeValue = node.value; + insert(node.id, parent, textElement, next); + break; + case "HTML": + let newDoc = doc.implementation.createHTMLDocument(""); + let docElement = newDoc.documentElement; + let pointer = doc.importNode(docElement, true); + doc.replaceChild(pointer, doc.documentElement); + if (doc.head) { doc.head.parentNode.removeChild(doc.head); } + if (doc.body) { doc.body.parentNode.removeChild(doc.body); } + nodes[node.id] = doc.documentElement; + break; + default: + let domElement = element(node.id); + domElement = domElement ? domElement : doc.createElement(node.tag); + node.attributes["data-id"] = `${node.id}`; + setAttributes(domElement as HTMLElement, node.attributes); + insert(node.id, parent, domElement, next); + break; + } + } +} + +function element(nodeId: number): Node { + return nodeId !== null && nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; +} + +function insert(id: number, parent: Node, node: Node, next: Node): void { + parent.insertBefore(node, next); + nodes[id] = node; +} + +function setAttributes(node: HTMLElement, attributes: object): void { + for (let attribute in node.attributes) { + if (node.hasAttribute(attribute)) { + node.removeAttribute(attribute); + } + } + for (let attribute in attributes) { + if (attributes[attribute]) { + node.setAttribute(attribute, attributes[attribute]); + } + } +} diff --git a/decode/dom.ts b/decode/dom.ts new file mode 100644 index 00000000..1a6f9534 --- /dev/null +++ b/decode/dom.ts @@ -0,0 +1,89 @@ +import { Token } from "@clarity-types/data"; +import { IDecodedNode } from "@clarity-types/dom"; +import { resolve } from "@src/data/token"; + +export default function(tokens: Token[]): Token[] { + let number = 0; + let lastType = null; + let node = []; + let decoded: Token[] = []; + let tagIndex = 0; + for (let token of tokens) { + let type = typeof(token); + switch (type) { + case "number": + if (type !== lastType && lastType !== null) { + decoded.push(process(node, tagIndex) as Token); + node = []; + tagIndex = 0; + } + number += token as number; + token = token === 0 ? token : number; + node.push(token); + tagIndex++; + break; + case "string": + node.push(token); + break; + case "object": + let subtoken = token[0]; + let subtype = typeof(subtoken); + switch (subtype) { + case "string": + let keys = resolve(token as string); + for (let key of keys) { + node.push(key); + } + break; + case "number": + token = tokens.length > subtoken ? tokens[subtoken] : null; + node.push(token); + break; + } + } + lastType = type; + } + + return decoded; +} + +function process(node: any[] | number[], tagIndex: number): IDecodedNode { + let output: IDecodedNode = { + id: node[0], + parent: tagIndex > 1 ? node[1] : null, + next: tagIndex > 2 ? node[2] : null, + tag: node[tagIndex] + }; + let hasAttribute = false; + let layouts = []; + let attributes = {}; + let value = null; + + for (let i = tagIndex + 1; i < node.length; i++) { + let token = node[i] as string; + let keyIndex = token.indexOf("="); + let parts = token.split("*"); + if (output.tag !== "*T" && keyIndex > 0) { + hasAttribute = true; + attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); + } else if (parts.length === 4) { + let layout = []; + for (let part of parts) { + layout.push(parseInt(part, 36)); + } + layouts.push(layout); + } else if (output.tag === "*T" && parts.length === 2) { + let textCount = parseInt(parts[0], 36); + let wordCount = parseInt(parts[1], 36); + value = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); + } else if (output.tag === "*T") { + value = token; + } + } + + if (layouts.length > 0) { output.layout = layouts; } + if (hasAttribute) { output.attributes = attributes; } + if (value) { output.value = value; } + + return output; +} diff --git a/src/clarity.ts b/src/clarity.ts index e65b8e96..76a38658 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,5 +1,4 @@ import * as event from "@src/core/event"; -import deserialize from "@src/data/deserialize"; import * as discover from "@src/dom/discover"; import * as mutation from "@src/dom/mutation"; import * as mouse from "@src/interactions/mouse"; @@ -8,8 +7,6 @@ import * as resize from "@src/viewport/resize"; import * as scroll from "@src/viewport/scroll"; import * as visibility from "@src/viewport/visibility"; -window["DESERIALIZE"] = deserialize; - /* Initial discovery of DOM */ export function start(): void { event.reset(); diff --git a/src/data/deserialize.ts b/src/data/deserialize.ts deleted file mode 100644 index c3fad563..00000000 --- a/src/data/deserialize.ts +++ /dev/null @@ -1,6 +0,0 @@ -import domDeserialize from "@src/dom/deserialize"; - -export default function(payload: string): string { - let json = JSON.parse(payload); - return domDeserialize(json.dom); -} diff --git a/src/dom/deserialize.ts b/src/dom/deserialize.ts deleted file mode 100644 index 1d2f775c..00000000 --- a/src/dom/deserialize.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { Token } from "@clarity-types/data"; -import { resolve } from "@src/data/token"; - -let nodes = {}; -let placeholder = document.createElement("iframe"); -// debug: visualize it on page -placeholder.height = "350"; -document.body.appendChild(placeholder); - -export default function(tokens: Token[]): string { - let number = 0; - let lastType = null; - let node = []; - let intermediate = []; - let tagIndex = 0; - for (let token of tokens) { - let type = typeof(token); - switch (type) { - case "number": - if (type !== lastType && lastType !== null) { - process(node, tagIndex); - intermediate.push(...node); - node = []; - tagIndex = 0; - } - number += token as number; - token = token === 0 ? token : number; - node.push(token); - tagIndex++; - break; - case "string": - node.push(token); - break; - case "object": - let subtoken = token[0]; - let subtype = typeof(subtoken); - switch (subtype) { - case "string": - let keys = resolve(token as string); - for (let key of keys) { - node.push(key); - } - break; - case "number": - token = tokens.length > subtoken ? tokens[subtoken] : null; - node.push(token); - break; - } - } - lastType = type; - } - - let markup = placeholder.contentDocument.documentElement.outerHTML; - - return JSON.stringify(intermediate) + "\r\n" + markup; -} - -function process(node: any[] | number[], tagIndex: number): void { - let id = node[0]; - let parent = tagIndex > 1 ? element(node[1]) : null; - let next = tagIndex > 2 ? element(node[2]) : null; - let layouts = []; - let tag = node[tagIndex]; - let doc = placeholder.contentDocument; - let content = null; - let attributes = {}; - - for (let i = tagIndex + 1; i < node.length; i++) { - let token = node[i] as string; - let keyIndex = token.indexOf("="); - let parts = token.split("*"); - if (tag !== "*T" && keyIndex > 0) { - attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); - } else if (parts.length === 4) { - let layout = []; - for (let part of parts) { - layout.push(parseInt(part, 36)); - } - layouts.push(layout); - } else if (tag === "*T" && parts.length === 2) { - let textCount = parseInt(parts[0], 36); - let wordCount = parseInt(parts[1], 36); - content = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); - } else if (tag === "*T") { - content = token; - } - } - - switch (tag) { - case "*D": - if (typeof XMLSerializer !== "undefined" && false) { - doc.open(); - doc.write(new XMLSerializer().serializeToString( - doc.implementation.createDocumentType( - attributes["name"], - attributes["publicId"], - attributes["systemId"] - ) - )); - doc.close(); - } - break; - case "*T": - let textElement = element(id); - textElement = textElement ? textElement : doc.createTextNode(null); - textElement.nodeValue = content; - insert(id, parent, textElement, next); - break; - case "HTML": - let newDoc = doc.implementation.createHTMLDocument(""); - let html = newDoc.documentElement; - let pointer = doc.importNode(html, true); - doc.replaceChild(pointer, doc.documentElement); - if (doc.head) { doc.head.parentNode.removeChild(doc.head); } - if (doc.body) { doc.body.parentNode.removeChild(doc.body); } - nodes[id] = doc.documentElement; - break; - default: - let domElement = element(id); - domElement = domElement ? domElement : doc.createElement(tag); - attributes["data-id"] = id; - setAttributes(domElement as HTMLElement, attributes); - insert(id, parent, domElement, next); - break; - } -} - -function element(nodeId: number): Node { - return nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; -} - -function insert(id: number, parent: Node, node: Node, next: Node): void { - parent.insertBefore(node, next); - nodes[id] = node; -} - -function setAttributes(node: HTMLElement, attributes: object): void { - for (let attribute in node.attributes) { - if (node.hasAttribute(attribute)) { - node.removeAttribute(attribute); - } - } - for (let attribute in attributes) { - if (attributes[attribute]) { - node.setAttribute(attribute, attributes[attribute]); - } - } -} diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 20cb493f..5f84ca19 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -2,7 +2,7 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; import queue from "@src/core/queue"; import time from "@src/core/time"; -import serialize from "@src/dom/serialize"; +import encode from "@src/dom/encode"; import * as timer from "@src/metrics/timer"; import processNode from "./node"; @@ -23,7 +23,7 @@ async function discover(): Promise { node = walker.nextNode(); } console.log("Finished discovering"); - let data = await serialize(method); + let data = await encode(method); timer.stop(method); return data; } diff --git a/src/dom/serialize.ts b/src/dom/encode.ts similarity index 100% rename from src/dom/serialize.ts rename to src/dom/encode.ts diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index e25d8ea3..22195d36 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -2,7 +2,7 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; import queue from "@src/core/queue"; import time from "@src/core/time"; -import serialize from "@src/dom/serialize"; +import encode from "@src/dom/encode"; import * as timer from "@src/metrics/timer"; import processNode from "./node"; @@ -69,7 +69,7 @@ async function process(mutations: MutationRecord[]): Promise { break; } } - let data = await serialize(method); + let data = await encode(method); timer.stop(method); return data; } diff --git a/src/interactions/serialize.ts b/src/interactions/encode.ts similarity index 100% rename from src/interactions/serialize.ts rename to src/interactions/encode.ts diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index b2f7b8de..3e6d3533 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -5,7 +5,7 @@ import { bind } from "@src/core/event"; import queue from "@src/core/queue"; import time from "@src/core/time"; import { getId } from "@src/dom/virtualdom"; -import serialize from "./serialize"; +import encode from "./encode"; let data: IMouseInteraction[] = []; let wait = config.lookahead; @@ -37,7 +37,7 @@ function handler(type: Mouse, evt: MouseEvent): void { } function schedule(): void { - queue(timestamp, Event.Mouse, serialize(Event.Mouse)); + queue(timestamp, Event.Mouse, encode(Event.Mouse)); } export function summarize(): IMouseInteraction[] { diff --git a/src/metrics/serialize.ts b/src/metrics/encode.ts similarity index 100% rename from src/metrics/serialize.ts rename to src/metrics/encode.ts diff --git a/src/metrics/metrics.ts b/src/metrics/metrics.ts index afba8a12..fc2f0806 100644 --- a/src/metrics/metrics.ts +++ b/src/metrics/metrics.ts @@ -1,8 +1,8 @@ import { Event, Flush } from "@clarity-types/data"; import queue from "@src/core/queue"; import time from "@src/core/time"; -import serialize from "./serialize"; +import encode from "./encode"; export function compute(): void { - queue(time(), Event.Metrics, serialize(), Flush.None); + queue(time(), Event.Metrics, encode(), Flush.None); } diff --git a/src/viewport/document.ts b/src/viewport/document.ts index bc788bac..c88e830e 100644 --- a/src/viewport/document.ts +++ b/src/viewport/document.ts @@ -2,7 +2,7 @@ import { Event, Flush } from "@clarity-types/data"; import { IDocumentSize } from "@clarity-types/viewport"; import queue from "@src/core/queue"; import time from "@src/core/time"; -import serialize from "./serialize"; +import encode from "./encode"; let data: IDocumentSize; @@ -28,7 +28,7 @@ function recompute(): void { updated: true }; - queue(time(), Event.Document, serialize(Event.Document), Flush.None); + queue(time(), Event.Document, encode(Event.Document), Flush.None); } export function compute(): void { diff --git a/src/viewport/serialize.ts b/src/viewport/encode.ts similarity index 100% rename from src/viewport/serialize.ts rename to src/viewport/encode.ts diff --git a/src/viewport/resize.ts b/src/viewport/resize.ts index 10012fff..b9b83174 100644 --- a/src/viewport/resize.ts +++ b/src/viewport/resize.ts @@ -3,7 +3,7 @@ import { IResizeViewport } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; import queue from "@src/core/queue"; import time from "@src/core/time"; -import serialize from "./serialize"; +import encode from "./encode"; let data: IResizeViewport; @@ -18,7 +18,7 @@ function recompute(): void { height: "innerHeight" in window ? window.innerHeight : document.documentElement.clientHeight, updated: true }; - queue(time(), Event.Resize, serialize(Event.Resize)); + queue(time(), Event.Resize, encode(Event.Resize)); } export function summarize(): IResizeViewport { diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts index 7d8bfbe6..a45562c0 100644 --- a/src/viewport/scroll.ts +++ b/src/viewport/scroll.ts @@ -3,7 +3,7 @@ import { IScrollViewport } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; import queue from "@src/core/queue"; import time from "@src/core/time"; -import serialize from "./serialize"; +import encode from "./encode"; let data: IScrollViewport[] = []; let wait = 1000; @@ -25,7 +25,7 @@ function recompute(): void { } function schedule(): void { - queue(timestamp, Event.Scroll, serialize(Event.Scroll)); + queue(timestamp, Event.Scroll, encode(Event.Scroll)); } export function summarize(): IScrollViewport[] { diff --git a/src/viewport/visibility.ts b/src/viewport/visibility.ts index 2dd9255d..97cfb869 100644 --- a/src/viewport/visibility.ts +++ b/src/viewport/visibility.ts @@ -3,7 +3,7 @@ import { IPageVisibility } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; import queue from "@src/core/queue"; import time from "@src/core/time"; -import serialize from "./serialize"; +import encode from "./encode"; let data: IPageVisibility; @@ -19,7 +19,7 @@ function recompute(): void { visible: "visibilityState" in document ? document.visibilityState : "default", updated: true }; - queue(time(), Event.Visibility, serialize(Event.Visibility)); + queue(time(), Event.Visibility, encode(Event.Visibility)); } export function summarize(): IPageVisibility { diff --git a/tsconfig.json b/tsconfig.json index 59038c1b..d5e3cdf3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,12 +9,14 @@ "baseUrl": ".", "paths": { "@src/*": ["src/*"], + "@decode/*": ["decode/*"], "@karma/*": ["karma/*"], "@clarity-types/*": ["types/*"] } }, "include":[ "src/**/*.ts", + "decode/**/*.ts", "types/**/*.d.ts", "karma/**/*.ts", ], diff --git a/types/data.d.ts b/types/data.d.ts index 962443ba..12f2e22c 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,4 +1,4 @@ -export type Token = (string | number | number[] | string[]); +export type Token = (any | string | number | number[] | string[]); export const enum Event { Discover, @@ -27,3 +27,9 @@ export interface IEvent { e: Event; d: Token[]; } + +export interface IDecodedEvent { + time: number; + event: Event; + data: Token[]; +} diff --git a/types/dom.d.ts b/types/dom.d.ts index 22bfe622..6512ce9d 100644 --- a/types/dom.d.ts +++ b/types/dom.d.ts @@ -18,3 +18,13 @@ export interface INodeValue { active?: boolean; update?: boolean; } + +export interface IDecodedNode { + id: number; + parent: number; + next: number; + tag: string; + attributes?: IAttributes; + layout?: number[][]; + value?: string; +} From da49bdf64a7f1fecaf7b827489421753e0e167ff Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 5 Aug 2019 07:06:37 -0700 Subject: [PATCH 019/105] Refactoring code to decode --- decode/clarity.ts | 10 +++++-- decode/dom.ts | 8 +++--- decode/metrics.ts | 56 ++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- src/core/config.ts | 4 ++- src/core/task.ts | 37 ++++++++++++++++++++++++++ src/data/upload.ts | 6 +++-- src/dom/discover.ts | 12 ++++----- src/dom/encode.ts | 7 ++--- src/dom/mutation.ts | 16 ++++++------ src/metrics/counter.ts | 2 +- src/metrics/encode.ts | 30 ++++++++++++--------- src/metrics/histogram.ts | 2 +- src/metrics/mark.ts | 2 +- src/metrics/timer.ts | 44 +++++++++---------------------- types/core.d.ts | 14 ++++++++++ types/data.d.ts | 5 ++-- types/metrics.d.ts | 14 ++++++---- webpack/configs/base.ts | 5 +++- webpack/configs/dev.ts | 2 +- webpack/configs/index.ts | 13 +++++++--- webpack/configs/prod.ts | 2 +- 22 files changed, 206 insertions(+), 87 deletions(-) create mode 100644 decode/metrics.ts create mode 100644 src/core/task.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index db353c3c..a06d3f23 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,6 +1,7 @@ -import { Event, IDecodedEvent, IEvent, Token } from "@clarity-types/data"; +import { DecodedToken, Event, IDecodedEvent, IEvent } from "@clarity-types/data"; import { IDecodedNode } from "@clarity-types/dom"; import dom from "@decode/dom"; +import metrics from "@decode/metrics"; let nodes = {}; let placeholder = document.createElement("iframe"); @@ -9,15 +10,20 @@ export function json(payload: string): IDecodedEvent[] { let decoded: IDecodedEvent[] = []; let encoded: IEvent[] = JSON.parse(payload); for (let entry of encoded) { - let exploded = { time: entry.t, event: entry.e, data: null as Token[] }; + let exploded = { time: entry.t, event: entry.e, data: entry.d as DecodedToken[] }; switch (entry.e) { case Event.Discover: case Event.Mutation: exploded.data = dom(entry.d); break; + case Event.Metrics: + exploded.data = metrics(entry.d); + break; } decoded.push(exploded); } + console.log("DECODED JSON Length: " + JSON.stringify(decoded).length); + console.log("DECODED JSON: " + JSON.stringify(decoded)); return decoded; } diff --git a/decode/dom.ts b/decode/dom.ts index 1a6f9534..d4da0579 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -1,19 +1,19 @@ -import { Token } from "@clarity-types/data"; +import { DecodedToken, Token } from "@clarity-types/data"; import { IDecodedNode } from "@clarity-types/dom"; import { resolve } from "@src/data/token"; -export default function(tokens: Token[]): Token[] { +export default function(tokens: Token[]): DecodedToken[] { let number = 0; let lastType = null; let node = []; - let decoded: Token[] = []; + let decoded: DecodedToken[] = []; let tagIndex = 0; for (let token of tokens) { let type = typeof(token); switch (type) { case "number": if (type !== lastType && lastType !== null) { - decoded.push(process(node, tagIndex) as Token); + decoded.push(process(node, tagIndex)); node = []; tagIndex = 0; } diff --git a/decode/metrics.ts b/decode/metrics.ts new file mode 100644 index 00000000..52f1a7f5 --- /dev/null +++ b/decode/metrics.ts @@ -0,0 +1,56 @@ +import { DecodedToken, Token } from "@clarity-types/data"; +import { Metric } from "@clarity-types/metrics"; + +export default function(tokens: Token[]): DecodedToken[] { + let lastType = null; + let metric = []; + let decoded: DecodedToken[] = []; + for (let token of tokens) { + let type = typeof(token); + switch (type) { + case "string": + if (type !== lastType && lastType !== null) { + decoded.push(process(metric)); + metric = []; + } + metric.push(token); + break; + case "number": + metric.push(token); + break; + } + lastType = type; + } + + return decoded; +} + +function process(metric: any[]): DecodedToken { + let name = metric[0]; + let type = name[name.length - 1]; + let output: DecodedToken = { metric: name, type }; + + switch (type) { + case Metric.Timer: + output["duration"] = metric[1]; + output["count"] = metric[2]; + break; + case Metric.Counter: + output["counter"] = metric[1]; + break; + case Metric.Histogram: + output["sum"] = metric[1]; + output["min"] = metric[2]; + output["max"] = metric[3]; + output["sumsquared"] = metric[4]; + output["count"] = metric[5]; + break; + case Metric.Mark: + output["tag"] = metric[1]; + output["start"] = metric[2]; + output["end"] = metric[3]; + break; + } + + return output; +} diff --git a/package.json b/package.json index 79d3886a..f431eeeb 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "webpack-merge": "^4.2.1" }, "scripts": { - "build": "yarn build:clean && yarn build:main && yarn build:cjs:dev", + "build": "yarn build:clean && yarn build:main && yarn build:cjs", "build:main": "webpack --config webpack/configs/index.ts", "build:cjs": "webpack --config webpack/configs/prod.ts", "build:cjs:dev": "webpack --config webpack/configs/dev.ts", diff --git a/src/core/config.ts b/src/core/config.ts index f1da18ad..b833976a 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,10 +1,12 @@ import { IConfig } from "@clarity-types/core"; export let config: IConfig = { + /* Core */ + longtask: 50, /* Interactions */ lookahead: 250, distance: 20, /* Data */ delay: 1000, - upload: null, + upload: null }; diff --git a/src/core/task.ts b/src/core/task.ts new file mode 100644 index 00000000..6a08b0c0 --- /dev/null +++ b/src/core/task.ts @@ -0,0 +1,37 @@ +import {ITask } from "@clarity-types/core"; +import { Timer } from "@clarity-types/metrics"; +import { config } from "@src/core/config"; +import * as timer from "@src/metrics/timer"; + +let tracker: ITask = {}; +let threshold = config.longtask; + +export function longtask(method: Timer): boolean { + let elapsed = Date.now() - tracker[method]; + return (elapsed > threshold); +} + +export function start(method: Timer): void { + if (!(method in tracker)) { + tracker[method] = 0; + } + tracker[method] = Date.now(); +} + +export function stop(method: Timer): void { + let end = Date.now(); + let duration = end - tracker[method]; + timer.observe(method, duration); +} + +export async function idle(method: Timer): Promise { + stop(method); + await wait(); + start(method); +} + +async function wait(): Promise { + return new Promise((resolve: FrameRequestCallback): void => { + requestAnimationFrame(resolve); + }); +} diff --git a/src/data/upload.ts b/src/data/upload.ts index 2064873c..750f7a5d 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,7 +1,9 @@ import { IEvent } from "@clarity-types/data"; +import * as decode from "@decode/clarity"; export default function(events: IEvent[]): void { let json = JSON.stringify(events); - console.log("Json: " + json); - window["PAYLOAD"].push(json); + console.log("JSON Length: " + JSON.stringify(json).length); + console.log("JSON: " + JSON.stringify(json)); + decode.json(json); } diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 5f84ca19..eac91723 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,9 +1,9 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; import queue from "@src/core/queue"; +import * as task from "@src/core/task"; import time from "@src/core/time"; import encode from "@src/dom/encode"; -import * as timer from "@src/metrics/timer"; import processNode from "./node"; export function start(): void { @@ -13,17 +13,17 @@ export function start(): void { } async function discover(): Promise { - let method = Timer.Discover; - timer.start(method); + let timer = Timer.Discover; + task.start(timer); let walker = document.createTreeWalker(document, NodeFilter.SHOW_ALL, null, false); let node = walker.nextNode(); while (node) { - if (timer.longtasks(method)) { await timer.idle(method); } + if (task.longtask(timer)) { await task.idle(timer); } processNode(node); node = walker.nextNode(); } console.log("Finished discovering"); - let data = await encode(method); - timer.stop(method); + let data = await encode(timer); + task.stop(timer); return data; } diff --git a/src/dom/encode.ts b/src/dom/encode.ts index 0da61b68..e912b7e5 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -1,21 +1,22 @@ import {Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; import { Counter, Timer } from "@clarity-types/metrics"; +import * as task from "@src/core/task"; import hash from "@src/data/hash"; import {check} from "@src/data/token"; import * as counter from "@src/metrics/counter"; -import * as timer from "@src/metrics/timer"; + import * as nodes from "./virtualdom"; window["HASH"] = hash; let reference: number = 0; -export default async function(method: Timer): Promise { +export default async function(timer: Timer): Promise { let markup = []; let values = nodes.summarize(); reference = 0; for (let value of values) { - if (timer.longtasks(method)) { await timer.idle(method); } + if (task.longtask(timer)) { await task.idle(timer); } let metadata = []; let layouts = []; let data: INodeData = value.data; diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 22195d36..dbe8a0fa 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,9 +1,9 @@ import { Event, Token } from "@clarity-types/data"; import { Timer } from "@clarity-types/metrics"; import queue from "@src/core/queue"; +import * as task from "@src/core/task"; import time from "@src/core/time"; import encode from "@src/dom/encode"; -import * as timer from "@src/metrics/timer"; import processNode from "./node"; let observer: MutationObserver; @@ -30,8 +30,8 @@ function handle(mutations: MutationRecord[]): void { } async function process(mutations: MutationRecord[]): Promise { - let method = Timer.Mutation; - timer.start(method); + let timer = Timer.Mutation; + task.start(timer); let length = mutations.length; for (let i = 0; i < length; i++) { let mutation = mutations[i]; @@ -43,7 +43,7 @@ async function process(mutations: MutationRecord[]): Promise { switch (mutation.type) { case "attributes": case "characterData": - if (timer.longtasks(method)) { await timer.idle(method); } + if (task.longtask(timer)) { await task.idle(timer); } processNode(target); break; case "childList": @@ -53,7 +53,7 @@ async function process(mutations: MutationRecord[]): Promise { let walker = document.createTreeWalker(mutation.addedNodes[j], NodeFilter.SHOW_ALL, null, false); let node = walker.currentNode; while (node) { - if (timer.longtasks(method)) { await timer.idle(method); } + if (task.longtask(timer)) { await task.idle(timer); } processNode(node); node = walker.nextNode(); } @@ -61,7 +61,7 @@ async function process(mutations: MutationRecord[]): Promise { // Process removes let removedLength = mutation.removedNodes.length; for (let j = 0; j < removedLength; j++) { - if (timer.longtasks(method)) { await timer.idle(method); } + if (task.longtask(timer)) { await task.idle(timer); } processNode(mutation.removedNodes[j]); } break; @@ -69,7 +69,7 @@ async function process(mutations: MutationRecord[]): Promise { break; } } - let data = await encode(method); - timer.stop(method); + let data = await encode(timer); + task.stop(timer); return data; } diff --git a/src/metrics/counter.ts b/src/metrics/counter.ts index 39957c59..d646f707 100644 --- a/src/metrics/counter.ts +++ b/src/metrics/counter.ts @@ -13,7 +13,7 @@ export function increment(key: Counter, counter: number = 1): void { export function summarize(): ICounterSummary { for (let key in tracker) { - if (tracker[key].updated) { + if (tracker[key]) { summary[key] = { counter: tracker[key].counter }; tracker[key].updated = false; } diff --git a/src/metrics/encode.ts b/src/metrics/encode.ts index 00a11d76..d9c129cc 100644 --- a/src/metrics/encode.ts +++ b/src/metrics/encode.ts @@ -7,42 +7,48 @@ import * as timer from "./timer"; export default function(): Token[] { let metrics = []; - // Serialize counters + // Encode counters let counters = counter.summarize(); for (let key in counters) { if (counters[key]) { let c = counters[key]; - metrics.push([key, num(c.counter)].join("*")); + metrics.push(key); + metrics.push(c.counter); } } - // Serialize histograms + // Encode histograms let histograms = histogram.summarize(); for (let key in histograms) { if (histograms[key]) { let h = histograms[key]; - metrics.push([key, num(h.sum), num(h.min), num(h.max), num(h.sumsquared), num(h.count)].join("*")); + metrics.push(key); + metrics.push(h.sum); + metrics.push(h.min); + metrics.push(h.max); + metrics.push(h.sumsquared); + metrics.push(h.count); } } - // Serialize timers + // Encode timers let timers = timer.summarize(); for (let key in timers) { if (timers[key]) { let t = timers[key]; - metrics.push([key, num(t.duration), num(t.count)].join("*")); + metrics.push(key); + metrics.push(t.duration); + metrics.push(t.count); } } - // Serialize marks + // Encode marks let marks = mark.summarize(); for (let m of marks) { - metrics.push([m.mark, num(m.start), num(m.end)].join("*")); + metrics.push(m.mark); + metrics.push(m.start); + metrics.push(m.end); } return metrics; } - -function num(x: number): string { - return x.toString(36); -} diff --git a/src/metrics/histogram.ts b/src/metrics/histogram.ts index 5867ea8d..4fabb72f 100644 --- a/src/metrics/histogram.ts +++ b/src/metrics/histogram.ts @@ -13,7 +13,7 @@ export function observe(key: Histogram, value: number): void { export function summarize(): IHistogramSummary { for (let key in tracker) { - if (tracker[key].updated) { + if (tracker[key]) { summary[key] = { sum: 0, min: -1, max: -1, sumsquared: 0, count: 0 }; for (let value of tracker[key].values) { summary[key].sum += value; diff --git a/src/metrics/mark.ts b/src/metrics/mark.ts index 39bfffe5..3008c4f1 100644 --- a/src/metrics/mark.ts +++ b/src/metrics/mark.ts @@ -10,7 +10,7 @@ export function mark(key: Mark, tag: string, start: number, end: number = 0): vo export function summarize(): IMarkSummary[] { for (let entry of tracker) { - if (entry.updated) { + if (entry) { summary.push({mark: entry.mark, start: entry.start, end: entry.end }); entry.updated = false; } diff --git a/src/metrics/timer.ts b/src/metrics/timer.ts index 33cb5e2f..b2e7d97a 100644 --- a/src/metrics/timer.ts +++ b/src/metrics/timer.ts @@ -1,44 +1,24 @@ -import {ITimer, ITimerSummary, Timer } from "@clarity-types/metrics"; +import { ITimer, ITimerSummary, Timer } from "@clarity-types/metrics"; let tracker: ITimer = {}; let summary: ITimerSummary = {}; -let threshold = 50; -window["TRACKER"] = tracker; // DEBUG: Remove later -export function longtasks(method: Timer): boolean { - let elapsed = Date.now() - tracker[method].start; - return (elapsed > threshold); -} - -export function start(method: Timer): void { - if (!(method in tracker)) { - tracker[method] = { start: 0, end: 0, duration: 0, count: 0 }; +export function observe(key: Timer, value: number): void { + if (!(key in tracker)) { + tracker[key] = { updated: true, values: [] }; } - tracker[method].start = Date.now(); -} - -export function stop(method: Timer): void { - tracker[method].end = Date.now(); - tracker[method].duration += tracker[method]["end"] - tracker[method]["start"]; - tracker[method].count++; -} - -export async function idle(method: Timer): Promise { - stop(method); - await wait(); - start(method); -} - -async function wait(): Promise { - return new Promise((resolve: FrameRequestCallback): void => { - requestAnimationFrame(resolve); - }); + tracker[key].updated = true; + tracker[key].values.push(value); } export function summarize(): ITimerSummary { for (let key in tracker) { - if (tracker[key].updated) { - summary[key] = { duration: tracker[key].duration, count: tracker[key].count }; + if (tracker[key]) { + summary[key] = { duration: 0, count: 0 }; + for (let value of tracker[key].values) { + summary[key].duration += value; + summary[key].count++; + } tracker[key].updated = false; } } diff --git a/types/core.d.ts b/types/core.d.ts index 99f76345..85cccf91 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -8,6 +8,8 @@ export interface IBindingContainer { } export interface IConfig { + /* Core */ + longtask: number; /* Interactions */ // Each interaction is going to wait until the specified milliseconds below before marking the end of interaction lookahead: number; @@ -19,3 +21,15 @@ export interface IConfig { // If left unspecified, raw payloads will be uploaded to the upload url endpoint upload: () => void; } + +// Task +export const enum Task { + Discover, + Mutation, + Wireup, + Active +} + +export interface ITask { + [key: number]: number; +} diff --git a/types/data.d.ts b/types/data.d.ts index 12f2e22c..953dbcd9 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,4 +1,5 @@ -export type Token = (any | string | number | number[] | string[]); +export type Token = (string | number | number[] | string[]); +export type DecodedToken = (any | any[]); export const enum Event { Discover, @@ -31,5 +32,5 @@ export interface IEvent { export interface IDecodedEvent { time: number; event: Event; - data: Token[]; + data: DecodedToken[]; } diff --git a/types/metrics.d.ts b/types/metrics.d.ts index d1f32280..d486af23 100644 --- a/types/metrics.d.ts +++ b/types/metrics.d.ts @@ -1,4 +1,11 @@ // Metric Names +export const enum Metric { + Timer = "t", + Counter = "c", + Histogram = "h", + Mark = "m" +} + export const enum Timer { Discover = "dt", Mutation = "mt", @@ -26,7 +33,7 @@ export const enum Histogram { export const enum Mark { Click = "cm", Error = "em", - Interaction = "ic" + Interaction = "im" } // Counter @@ -90,10 +97,7 @@ export interface ITimer { interface ITimerValue { updated: boolean; - start: number; - end: number; - duration: number; - count: number; + values: [number]; } export interface ITimerSummary { diff --git a/webpack/configs/base.ts b/webpack/configs/base.ts index a70f2740..1a0425cf 100644 --- a/webpack/configs/base.ts +++ b/webpack/configs/base.ts @@ -6,7 +6,10 @@ import { TsConfigPathsPlugin } from "awesome-typescript-loader"; // https://webpack.js.org/configuration const CommongConfig: webpack.Configuration = { - entry: "./webpack/globalize.ts", + entry: { + clarity: "./webpack/globalize.ts", + decode: "./decode/clarity.ts" + }, output: { path: `${__dirname}/../../build` diff --git a/webpack/configs/dev.ts b/webpack/configs/dev.ts index 790c774c..bb3f3ffd 100644 --- a/webpack/configs/dev.ts +++ b/webpack/configs/dev.ts @@ -10,7 +10,7 @@ const DevConfig: webpack.Configuration = { mode: "development", output: { - filename: "clarity.js" + filename: "[name].js" }, devtool: "inline-source-map" diff --git a/webpack/configs/index.ts b/webpack/configs/index.ts index 8f508cb3..82b953db 100644 --- a/webpack/configs/index.ts +++ b/webpack/configs/index.ts @@ -9,15 +9,22 @@ const IndexConfig: webpack.Configuration = { mode: "production", - entry: "./src/index.ts", + entry: { + clarity: "./webpack/globalize.ts", + decode: "./decode/clarity.ts" + }, output: { libraryTarget: "commonjs", - filename: "index.js" + filename: "[name].js" }, optimization: { - minimize: false + minimize: false, + splitChunks: { + // include all types of chunks + chunks: "all" + } } }; diff --git a/webpack/configs/prod.ts b/webpack/configs/prod.ts index 38cda54d..14664616 100644 --- a/webpack/configs/prod.ts +++ b/webpack/configs/prod.ts @@ -11,7 +11,7 @@ const ProdConfig: webpack.Configuration = { mode: "production", output: { - filename: "clarity.min.js" + filename: "[name].min.js" }, optimization: { From bcf172919c9a9aae1d6819eef3445ec91e1e3aee Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 5 Aug 2019 07:47:52 -0700 Subject: [PATCH 020/105] Updating webpack configuration --- types/index.d.ts | 4 ++++ webpack/configs/base.ts | 5 ----- webpack/configs/dev.ts | 7 ++++++- webpack/configs/index.ts | 2 +- webpack/configs/prod.ts | 5 +++++ 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/types/index.d.ts b/types/index.d.ts index 60c1c2ff..71e19794 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -1,5 +1,7 @@ interface IClarityJs { version: string; + start: () => void; + end: () => void; } declare const ClarityJs: IClarityJs; @@ -7,3 +9,5 @@ declare const ClarityJs: IClarityJs; export * from "./data"; export * from "./dom"; export * from "./metrics"; + +export { ClarityJs }; diff --git a/webpack/configs/base.ts b/webpack/configs/base.ts index 1a0425cf..b9ff2735 100644 --- a/webpack/configs/base.ts +++ b/webpack/configs/base.ts @@ -6,11 +6,6 @@ import { TsConfigPathsPlugin } from "awesome-typescript-loader"; // https://webpack.js.org/configuration const CommongConfig: webpack.Configuration = { - entry: { - clarity: "./webpack/globalize.ts", - decode: "./decode/clarity.ts" - }, - output: { path: `${__dirname}/../../build` }, diff --git a/webpack/configs/dev.ts b/webpack/configs/dev.ts index bb3f3ffd..8ce5db2d 100644 --- a/webpack/configs/dev.ts +++ b/webpack/configs/dev.ts @@ -9,8 +9,13 @@ const DevConfig: webpack.Configuration = { mode: "development", + entry: { + clarity: "./webpack/globalize.ts", + decode: "./decode/clarity.ts" + }, + output: { - filename: "[name].js" + filename: "[name].dev.js" }, devtool: "inline-source-map" diff --git a/webpack/configs/index.ts b/webpack/configs/index.ts index 82b953db..de8f26ad 100644 --- a/webpack/configs/index.ts +++ b/webpack/configs/index.ts @@ -10,7 +10,7 @@ const IndexConfig: webpack.Configuration = { mode: "production", entry: { - clarity: "./webpack/globalize.ts", + index: "./src/index.ts", decode: "./decode/clarity.ts" }, diff --git a/webpack/configs/prod.ts b/webpack/configs/prod.ts index 14664616..999d425b 100644 --- a/webpack/configs/prod.ts +++ b/webpack/configs/prod.ts @@ -10,6 +10,11 @@ const ProdConfig: webpack.Configuration = { mode: "production", + entry: { + clarity: "./webpack/globalize.ts", + decode: "./decode/clarity.ts" + }, + output: { filename: "[name].min.js" }, From b3aa40aa3bfec7fc0a48c7a5d0dca3a17179de37 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 6 Aug 2019 09:18:59 -0700 Subject: [PATCH 021/105] Streamlining configuration and updating decode imports --- decode/clarity.ts | 8 ++++---- decode/dom.ts | 6 +++--- decode/index.ts | 1 + decode/metrics.ts | 4 ++-- src/clarity.ts | 12 +++++++++--- src/core/config.ts | 9 ++++----- src/core/task.ts | 2 +- src/data/upload.ts | 14 ++++++++++---- src/index.ts | 4 ++-- src/interactions/mouse.ts | 2 +- types/clarity.d.ts | 4 ---- types/core.d.ts | 17 +++++------------ types/index.d.ts | 10 +++++++--- 13 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 decode/index.ts delete mode 100644 types/clarity.d.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index a06d3f23..3944a2ed 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,7 +1,7 @@ -import { DecodedToken, Event, IDecodedEvent, IEvent } from "@clarity-types/data"; -import { IDecodedNode } from "@clarity-types/dom"; -import dom from "@decode/dom"; -import metrics from "@decode/metrics"; +import { DecodedToken, Event, IDecodedEvent, IEvent } from "../types/data"; +import { IDecodedNode } from "../types/dom"; +import dom from "./dom"; +import metrics from "./metrics"; let nodes = {}; let placeholder = document.createElement("iframe"); diff --git a/decode/dom.ts b/decode/dom.ts index d4da0579..cfacb26f 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -1,6 +1,6 @@ -import { DecodedToken, Token } from "@clarity-types/data"; -import { IDecodedNode } from "@clarity-types/dom"; -import { resolve } from "@src/data/token"; +import { resolve } from "../src/data/token"; +import { DecodedToken, Token } from "../types/data"; +import { IDecodedNode } from "../types/dom"; export default function(tokens: Token[]): DecodedToken[] { let number = 0; diff --git a/decode/index.ts b/decode/index.ts new file mode 100644 index 00000000..7e423511 --- /dev/null +++ b/decode/index.ts @@ -0,0 +1 @@ +export * from "./clarity"; diff --git a/decode/metrics.ts b/decode/metrics.ts index 52f1a7f5..52042c6f 100644 --- a/decode/metrics.ts +++ b/decode/metrics.ts @@ -1,5 +1,5 @@ -import { DecodedToken, Token } from "@clarity-types/data"; -import { Metric } from "@clarity-types/metrics"; +import { DecodedToken, Token } from "../types/data"; +import { Metric } from "../types/metrics"; export default function(tokens: Token[]): DecodedToken[] { let lastType = null; diff --git a/src/clarity.ts b/src/clarity.ts index 76a38658..e28d33ee 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,3 +1,5 @@ +import { IConfig } from "@clarity-types/core"; +import config from "@src/core/config"; import * as event from "@src/core/event"; import * as discover from "@src/dom/discover"; import * as mutation from "@src/dom/mutation"; @@ -8,7 +10,13 @@ import * as scroll from "@src/viewport/scroll"; import * as visibility from "@src/viewport/visibility"; /* Initial discovery of DOM */ -export function start(): void { +export function start(configuration: IConfig = {}): void { + + // Process custom configuration, if available + for (let key in configuration) { + if (key in config) { config[key] = configuration[key]; } + } + event.reset(); // DOM @@ -30,5 +38,3 @@ export function end(): void { event.reset(); mutation.end(); } - -start(); diff --git a/src/core/config.ts b/src/core/config.ts index b833976a..b3a1ddee 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,12 +1,11 @@ import { IConfig } from "@clarity-types/core"; -export let config: IConfig = { - /* Core */ +let config: IConfig = { longtask: 50, - /* Interactions */ lookahead: 250, distance: 20, - /* Data */ - delay: 1000, + delay: 250, upload: null }; + +export default config; diff --git a/src/core/task.ts b/src/core/task.ts index 6a08b0c0..afd53b50 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,6 +1,6 @@ import {ITask } from "@clarity-types/core"; import { Timer } from "@clarity-types/metrics"; -import { config } from "@src/core/config"; +import config from "@src/core/config"; import * as timer from "@src/metrics/timer"; let tracker: ITask = {}; diff --git a/src/data/upload.ts b/src/data/upload.ts index 750f7a5d..b8e134f9 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,9 +1,15 @@ import { IEvent } from "@clarity-types/data"; import * as decode from "@decode/clarity"; +import config from "@src/core/config"; export default function(events: IEvent[]): void { - let json = JSON.stringify(events); - console.log("JSON Length: " + JSON.stringify(json).length); - console.log("JSON: " + JSON.stringify(json)); - decode.json(json); + let payload = JSON.stringify(events); + let upload = config.upload ? config.upload : send; + upload(payload); +} + +function send(payload: string): void { + console.log("JSON Length: " + JSON.stringify(payload).length); + console.log("JSON: " + JSON.stringify(payload)); + decode.json(payload); } diff --git a/src/index.ts b/src/index.ts index d84e780f..e81ae3fe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,3 @@ -import * as ClarityJs from "./clarity"; +import * as clarity from "./clarity"; -export { ClarityJs }; +export { clarity }; diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index 3e6d3533..eebea43a 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -1,6 +1,6 @@ import { Event } from "@clarity-types/data"; import { IMouseInteraction, Mouse } from "@clarity-types/interactions"; -import { config } from "@src/core/config"; +import config from "@src/core/config"; import { bind } from "@src/core/event"; import queue from "@src/core/queue"; import time from "@src/core/time"; diff --git a/types/clarity.d.ts b/types/clarity.d.ts deleted file mode 100644 index a84477ed..00000000 --- a/types/clarity.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface IClarity { - start: () => void; - end: () => void; -} diff --git a/types/core.d.ts b/types/core.d.ts index 85cccf91..4477f615 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -8,18 +8,11 @@ export interface IBindingContainer { } export interface IConfig { - /* Core */ - longtask: number; - /* Interactions */ - // Each interaction is going to wait until the specified milliseconds below before marking the end of interaction - lookahead: number; - distance: number; - /* Data */ - // Each new event is going to delay data upload to server by this number of milliseconds - delay: number; - // Pointer to the function which would be responsible for sending the data - // If left unspecified, raw payloads will be uploaded to the upload url endpoint - upload: () => void; + longtask?: number; + lookahead?: number; + distance?: number; + delay?: number; + upload?: (payload: string) => void; } // Task diff --git a/types/index.d.ts b/types/index.d.ts index 71e19794..778bf7be 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -1,13 +1,17 @@ +import { IConfig } from "./core"; + interface IClarityJs { version: string; - start: () => void; + start: (config: IConfig) => void; end: () => void; } -declare const ClarityJs: IClarityJs; +declare const clarity: IClarityJs; export * from "./data"; export * from "./dom"; +export * from "./interactions"; export * from "./metrics"; +export * from "./viewport"; -export { ClarityJs }; +export { clarity }; From ca6e7bf1748044bcf14cfc2b52c5f79289d3d376 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 6 Aug 2019 15:23:24 -0700 Subject: [PATCH 022/105] Removing console statements --- decode/clarity.ts | 2 -- src/data/upload.ts | 2 -- src/dom/discover.ts | 1 - src/dom/virtualdom.ts | 5 ----- 4 files changed, 10 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 3944a2ed..4080cc22 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -22,8 +22,6 @@ export function json(payload: string): IDecodedEvent[] { } decoded.push(exploded); } - console.log("DECODED JSON Length: " + JSON.stringify(decoded).length); - console.log("DECODED JSON: " + JSON.stringify(decoded)); return decoded; } diff --git a/src/data/upload.ts b/src/data/upload.ts index b8e134f9..cd451896 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -9,7 +9,5 @@ export default function(events: IEvent[]): void { } function send(payload: string): void { - console.log("JSON Length: " + JSON.stringify(payload).length); - console.log("JSON: " + JSON.stringify(payload)); decode.json(payload); } diff --git a/src/dom/discover.ts b/src/dom/discover.ts index eac91723..8626d0d0 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -22,7 +22,6 @@ async function discover(): Promise { processNode(node); node = walker.nextNode(); } - console.log("Finished discovering"); let data = await encode(timer); task.stop(timer); return data; diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 87a77613..bc7407e7 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -42,26 +42,21 @@ export function add(node: Node, data: INodeData): void { export function update(node: Node, data: INodeData): void { let id = getId(node); - console.log("Updating node: " + id); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = node.nextSibling ? getId(node.nextSibling) : null; if (id in values) { let value = values[id]; - console.log("Previous value: " + JSON.stringify(value)); // Handle case where internal ordering may have changed if (value["next"] !== nextId) { - let oldNextId = value["next"]; value["next"] = nextId; - console.log("Old next id: " + oldNextId + " | " + nextId); } // Handle case where parent might have been updated if (value["parent"] !== parentId) { let oldParentId = value["parent"]; value["parent"] = parentId; - console.log("Old parent id: " + oldParentId + " | " + parentId); // Move this node to the right location under new parent if (parentId >= 0) { if (nextId >= 0) { From 61eca382bb3286362743a8187a7d341c548cb19a Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 6 Aug 2019 15:28:55 -0700 Subject: [PATCH 023/105] Remove console statements --- src/dom/mutation.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index dbe8a0fa..4fac1959 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -37,9 +37,6 @@ async function process(mutations: MutationRecord[]): Promise { let mutation = mutations[i]; let target = mutation.target; - console.log("Received mutation: " + mutation.type + " | " + mutation.target); - window["MUTATIONS"].push(mutation); - switch (mutation.type) { case "attributes": case "characterData": From c1693c9c75bb2ef406f2a55062953ce5b4fa2d8f Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 7 Aug 2019 09:17:28 -0700 Subject: [PATCH 024/105] Adding some debug statements and fixing tokens --- decode/clarity.ts | 36 ++++++++++++++++++++++++++++-------- decode/dom.ts | 3 ++- src/clarity.ts | 3 +++ src/core/config.ts | 1 + src/data/token.ts | 9 ++++----- src/data/track.ts | 42 ++++++++++++++++++++++++++++++++++++++++++ src/data/upload.ts | 13 ++++++++++--- src/dom/encode.ts | 2 +- src/dom/mutation.ts | 1 + src/dom/virtualdom.ts | 5 ++++- types/core.d.ts | 5 ++++- types/data.d.ts | 8 ++++++++ 12 files changed, 108 insertions(+), 20 deletions(-) create mode 100644 src/data/track.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 4080cc22..7dcba150 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,19 +1,32 @@ -import { DecodedToken, Event, IDecodedEvent, IEvent } from "../types/data"; +import { DecodedToken, Event, IDecodedEvent, IEvent, IPayload } from "../types/data"; import { IDecodedNode } from "../types/dom"; import dom from "./dom"; import metrics from "./metrics"; let nodes = {}; -let placeholder = document.createElement("iframe"); +let pageId: string = null; +let payloads: IPayload[] = []; + +export function json(payload: IPayload): IDecodedEvent[] { + if (pageId !== payload.p) { + payloads = []; + nodes = {}; + pageId = payload.p; + } -export function json(payload: string): IDecodedEvent[] { let decoded: IDecodedEvent[] = []; - let encoded: IEvent[] = JSON.parse(payload); + let encoded: IEvent[] = JSON.parse(payload.d); + payloads.push(payload); + let count = 0; for (let entry of encoded) { let exploded = { time: entry.t, event: entry.e, data: entry.d as DecodedToken[] }; + count++; switch (entry.e) { case Event.Discover: case Event.Mutation: + console.warn("!Event: " + (entry.e === Event.Mutation ? "Mutation" : "Discover") + + " | Payload #" + payload.n + " | Event #" + count + " | Nodes #" + entry.d.length + + " | First Id: " + (entry.d.length > 0 ? entry.d[0] : -1) + " | Events: " + JSON.stringify(entry.d)); exploded.data = dom(entry.d); break; case Event.Metrics: @@ -25,22 +38,28 @@ export function json(payload: string): IDecodedEvent[] { return decoded; } -export function html(payload: string): string { +export function html(payload: IPayload): string { + let placeholder = document.createElement("iframe"); + render(payload, placeholder); + return placeholder.contentDocument.documentElement.outerHTML; +} + +export function render(payload: IPayload, placeholder: HTMLIFrameElement): void { let decoded = json(payload); for (let entry of decoded) { switch (entry.event) { case Event.Discover: case Event.Mutation: - markup(entry.data); + markup(entry.data, placeholder); break; } } - return placeholder.contentDocument.documentElement.outerHTML; } -function markup(data: IDecodedNode[]): void { +function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { let doc = placeholder.contentDocument; for (let node of data) { + console.log("Markup Node: " + JSON.stringify(node)); let parent = element(node.parent); let next = element(node.next); switch (node.tag) { @@ -75,6 +94,7 @@ function markup(data: IDecodedNode[]): void { default: let domElement = element(node.id); domElement = domElement ? domElement : doc.createElement(node.tag); + if (!node.attributes) { node.attributes = {}; } node.attributes["data-id"] = `${node.id}`; setAttributes(domElement as HTMLElement, node.attributes); insert(node.id, parent, domElement, next); diff --git a/decode/dom.ts b/decode/dom.ts index cfacb26f..2f6e92c5 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -18,7 +18,7 @@ export default function(tokens: Token[]): DecodedToken[] { tagIndex = 0; } number += token as number; - token = token === 0 ? token : number; + token = number; node.push(token); tagIndex++; break; @@ -85,5 +85,6 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { if (hasAttribute) { output.attributes = attributes; } if (value) { output.value = value; } + console.log("Node: " + JSON.stringify(output)); return output; } diff --git a/src/clarity.ts b/src/clarity.ts index e28d33ee..de12e021 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,6 +1,7 @@ import { IConfig } from "@clarity-types/core"; import config from "@src/core/config"; import * as event from "@src/core/event"; +import * as track from "@src/data/track"; import * as discover from "@src/dom/discover"; import * as mutation from "@src/dom/mutation"; import * as mouse from "@src/interactions/mouse"; @@ -18,6 +19,7 @@ export function start(configuration: IConfig = {}): void { } event.reset(); + track.start(); // DOM mutation.start(); @@ -37,4 +39,5 @@ export function start(configuration: IConfig = {}): void { export function end(): void { event.reset(); mutation.end(); + track.end(); } diff --git a/src/core/config.ts b/src/core/config.ts index b3a1ddee..d3c333fe 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -5,6 +5,7 @@ let config: IConfig = { lookahead: 250, distance: 20, delay: 250, + tokens: [], upload: null }; diff --git a/src/data/token.ts b/src/data/token.ts index 8f1bfbd6..7042b687 100644 --- a/src/data/token.ts +++ b/src/data/token.ts @@ -1,11 +1,10 @@ -let tokens = {}; +let tokens: string[] = []; -export function check(hash: string, metadata: string[]): boolean { - let output = hash in tokens; - tokens[hash] = metadata; +export function check(hash: string): boolean { + let output = tokens.indexOf(hash) >= 0; return output; } export function resolve(hash: string): string[] { - return hash in tokens ? tokens[hash] : []; + return check(hash) ? tokens[hash] : []; } diff --git a/src/data/track.ts b/src/data/track.ts new file mode 100644 index 00000000..8e29df0b --- /dev/null +++ b/src/data/track.ts @@ -0,0 +1,42 @@ +import hash from "@src/data/hash"; + +export let pageId: string; +export let sessionId: string; +export let tenantId: string; +let count: number = 0; + +export function start(): void { + pageId = guid(); + sessionId = guid(); + tenantId = hash(top.location.host); + count = 0; +} + +export function end(): void { + pageId = null; + sessionId = null; + tenantId = null; + count = 0; +} + +export function sequence(): number { + return count++; +} + +// Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript +// Excluding 3rd party code from tslint +// tslint:disable +function guid() { + let d = new Date().getTime(); + if (window.performance && performance.now) { + // Use high-precision timer if available + d += performance.now(); + } + let uuid = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, function(c) { + let r = (d + Math.random() * 16) % 16 | 0; + d = Math.floor(d / 16); + return (c == "x" ? r : (r & 0x3 | 0x8)).toString(16); + }); + return uuid; +} +// tslint:enable diff --git a/src/data/upload.ts b/src/data/upload.ts index cd451896..50bac306 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,13 +1,20 @@ -import { IEvent } from "@clarity-types/data"; +import { IEvent, IPayload } from "@clarity-types/data"; import * as decode from "@decode/clarity"; import config from "@src/core/config"; +import * as track from "@src/data/track"; export default function(events: IEvent[]): void { - let payload = JSON.stringify(events); + let payload: IPayload = { + p: track.pageId, + s: track.sessionId, + t: track.tenantId, + n: track.sequence(), + d: JSON.stringify(events) + }; let upload = config.upload ? config.upload : send; upload(payload); } -function send(payload: string): void { +function send(payload: IPayload): void { decode.json(payload); } diff --git a/src/dom/encode.ts b/src/dom/encode.ts index e912b7e5..4384f0d7 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -72,7 +72,7 @@ export default async function(timer: Timer): Promise { function meta(metadata: string[]): string[] | string[][] { let value = JSON.stringify(metadata); let hashed = hash(value); - return check(hashed, metadata) && hashed.length < value.length ? [[hashed]] : metadata; + return check(hashed) && hashed.length < value.length ? [[hashed]] : metadata; } function text(tag: string, value: string): string { diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 4fac1959..1b7980a4 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -24,6 +24,7 @@ export function end(): void { } function handle(mutations: MutationRecord[]): void { + window["MUTATIONS"].push(mutations); process(mutations).then((data: Token[]) => { queue(time(), Event.Mutation, data); }); diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index bc7407e7..28ecf4eb 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -10,6 +10,9 @@ let backupIndex: number; let backupNodes: Node[]; let backupValues: Node[]; +// For debugging +window["DOM"] = { getId, get }; + export function getId(node: Node, autogen: boolean = true): number { if (node === null) { return null; } let id = node[NODE_ID_PROP]; @@ -58,7 +61,7 @@ export function update(node: Node, data: INodeData): void { let oldParentId = value["parent"]; value["parent"] = parentId; // Move this node to the right location under new parent - if (parentId >= 0) { + if (parentId !== null && parentId >= 0) { if (nextId >= 0) { values[parentId].children.splice(nextId + 1, 0 , id); } else { diff --git a/types/core.d.ts b/types/core.d.ts index 4477f615..5cc7ae4a 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -1,3 +1,5 @@ +import { IPayload } from "./data"; + export interface IEventBindingPair { target: EventTarget; listener: EventListener; @@ -12,7 +14,8 @@ export interface IConfig { lookahead?: number; distance?: number; delay?: number; - upload?: (payload: string) => void; + tokens?: string[]; + upload?: (payload: IPayload) => void; } // Task diff --git a/types/data.d.ts b/types/data.d.ts index 953dbcd9..08868e39 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -29,6 +29,14 @@ export interface IEvent { d: Token[]; } +export interface IPayload { + p: string; + s: string; + t: string; + n: number; + d: string; +} + export interface IDecodedEvent { time: number; event: Event; From aee28f2d944078ec9eceb71b4801bb7d595f0801 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 7 Aug 2019 16:37:39 -0700 Subject: [PATCH 025/105] Bug fixes and adding support for devtools tracking --- decode/clarity.ts | 2 +- src/dom/discover.ts | 3 ++- src/dom/mutation.ts | 11 ++++++++--- src/dom/node.ts | 12 ++++++++---- src/dom/virtualdom.ts | 15 ++++++++++----- types/dom.d.ts | 15 +++++++++++++++ 6 files changed, 44 insertions(+), 14 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 7dcba150..4a835beb 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -108,7 +108,7 @@ function element(nodeId: number): Node { } function insert(id: number, parent: Node, node: Node, next: Node): void { - parent.insertBefore(node, next); + if (parent !== null) { parent.insertBefore(node, next); } nodes[id] = node; } diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 8626d0d0..572caf5a 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,4 +1,5 @@ import { Event, Token } from "@clarity-types/data"; +import { Source } from "@clarity-types/dom"; import { Timer } from "@clarity-types/metrics"; import queue from "@src/core/queue"; import * as task from "@src/core/task"; @@ -19,7 +20,7 @@ async function discover(): Promise { let node = walker.nextNode(); while (node) { if (task.longtask(timer)) { await task.idle(timer); } - processNode(node); + processNode(node, Source.Discover); node = walker.nextNode(); } let data = await encode(timer); diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 1b7980a4..ba60f002 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,4 +1,5 @@ import { Event, Token } from "@clarity-types/data"; +import { Source } from "@clarity-types/dom"; import { Timer } from "@clarity-types/metrics"; import queue from "@src/core/queue"; import * as task from "@src/core/task"; @@ -24,6 +25,7 @@ export function end(): void { } function handle(mutations: MutationRecord[]): void { + window["MUTATIONS"].push(time()); window["MUTATIONS"].push(mutations); process(mutations).then((data: Token[]) => { queue(time(), Event.Mutation, data); @@ -40,9 +42,12 @@ async function process(mutations: MutationRecord[]): Promise { switch (mutation.type) { case "attributes": + if (task.longtask(timer)) { await task.idle(timer); } + processNode(target, Source.Attributes); + break; case "characterData": if (task.longtask(timer)) { await task.idle(timer); } - processNode(target); + processNode(target, Source.CharacterData); break; case "childList": // Process additions @@ -52,7 +57,7 @@ async function process(mutations: MutationRecord[]): Promise { let node = walker.currentNode; while (node) { if (task.longtask(timer)) { await task.idle(timer); } - processNode(node); + processNode(node, Source.ChildListAdd); node = walker.nextNode(); } } @@ -60,7 +65,7 @@ async function process(mutations: MutationRecord[]): Promise { let removedLength = mutation.removedNodes.length; for (let j = 0; j < removedLength; j++) { if (task.longtask(timer)) { await task.idle(timer); } - processNode(mutation.removedNodes[j]); + processNode(mutation.removedNodes[j], Source.ChildListRemove); } break; default: diff --git a/src/dom/node.ts b/src/dom/node.ts index 7af853e1..073cc57a 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -1,15 +1,19 @@ +import { Source } from "@clarity-types/dom"; import * as nodes from "./virtualdom"; let ignoreAttributes = ["title", "alt", "onload", "onfocus"]; -export default function(node: Node): void { +export default function(node: Node, source: Source): void { + // Do not track this change if we are attempting to remove a node before discovering it + if (source === Source.ChildListRemove && nodes.has(node) === false) { return; } + let call = nodes.has(node) ? "update" : "add"; switch (node.nodeType) { case Node.DOCUMENT_TYPE_NODE: let doctype = node as DocumentType; let docAttributes = { name: doctype.name, publicId: doctype.publicId, systemId: doctype.systemId }; let docData = { tag: "*D", attributes: docAttributes }; - nodes[call](node, docData); + nodes[call](node, docData, source); break; case Node.TEXT_NODE: // Account for this text node only if we are tracking the parent node @@ -18,7 +22,7 @@ export default function(node: Node): void { if (parent && nodes.has(parent)) { let textData = { tag: "*T", value: node.nodeValue }; textData["layout"] = getTextLayout(node); - nodes[call](node, textData); + nodes[call](node, textData, source); } break; case Node.ELEMENT_NODE: @@ -31,7 +35,7 @@ export default function(node: Node): void { default: let data = { tag: element.tagName, attributes: getAttributes(element.attributes) }; data["layout"] = getLayout(element); - nodes[call](node, data); + nodes[call](node, data, source); break; } break; diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 28ecf4eb..37de936c 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -1,4 +1,5 @@ -import {INodeData, INodeValue } from "@clarity-types/dom"; +import { Action, INodeData, INodeValue, Source } from "@clarity-types/dom"; +import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; let index: number = 1; @@ -11,7 +12,7 @@ let backupNodes: Node[]; let backupValues: Node[]; // For debugging -window["DOM"] = { getId, get }; +window["DOM"] = { getId, get, getNode }; export function getId(node: Node, autogen: boolean = true): number { if (node === null) { return null; } @@ -22,7 +23,7 @@ export function getId(node: Node, autogen: boolean = true): number { return id ? id : null; } -export function add(node: Node, data: INodeData): void { +export function add(node: Node, data: INodeData, source: Source): void { let id = getId(node); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = node.nextSibling ? getId(node.nextSibling, false) : null; @@ -39,11 +40,12 @@ export function add(node: Node, data: INodeData): void { children: [], active: true, update: true, + track: [time(), Action.Add, source], data }; } -export function update(node: Node, data: INodeData): void { +export function update(node: Node, data: INodeData, source: Source): void { let id = getId(node); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = node.nextSibling ? getId(node.nextSibling) : null; @@ -87,6 +89,9 @@ export function update(node: Node, data: INodeData): void { } value["update"] = true; + value["track"].push(time()); + value["track"].push(Action.Update); + value["track"].push(source); } } @@ -103,7 +108,7 @@ export function get(node: Node): INodeValue { } export function has(node: Node): boolean { - return getId(node) in nodes; + return getId(node, false) in nodes; } export function remove(node: Node): void { diff --git a/types/dom.d.ts b/types/dom.d.ts index 6512ce9d..dcd07dc7 100644 --- a/types/dom.d.ts +++ b/types/dom.d.ts @@ -1,3 +1,17 @@ +export const enum Source { + Discover, + ChildListAdd, + ChildListRemove, + Attributes, + CharacterData +} + +export const enum Action { + Add, + Update, + Remove +} + export interface IAttributes { [key: string]: string; } @@ -15,6 +29,7 @@ export interface INodeValue { next: number; children: number[]; data: INodeData; + track: number[]; active?: boolean; update?: boolean; } From e726d99cacc49720ef31acbd0f3007b107bc70c1 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 8 Aug 2019 08:18:52 -0700 Subject: [PATCH 026/105] Bug fixes and adding decode support for viewport --- decode/clarity.ts | 43 +++++++++++++++++++++++++++++++++++++------ decode/dom.ts | 7 +++++-- decode/metrics.ts | 4 ++-- decode/viewport.ts | 19 +++++++++++++++++++ src/core/config.ts | 1 + src/dom/virtualdom.ts | 21 ++++++++++++++------- types/core.d.ts | 1 + types/dom.d.ts | 8 +------- 8 files changed, 80 insertions(+), 24 deletions(-) create mode 100644 decode/viewport.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 4a835beb..0a3b99a6 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -2,6 +2,7 @@ import { DecodedToken, Event, IDecodedEvent, IEvent, IPayload } from "../types/d import { IDecodedNode } from "../types/dom"; import dom from "./dom"; import metrics from "./metrics"; +import viewport from "./viewport"; let nodes = {}; let pageId: string = null; @@ -22,15 +23,20 @@ export function json(payload: IPayload): IDecodedEvent[] { let exploded = { time: entry.t, event: entry.e, data: entry.d as DecodedToken[] }; count++; switch (entry.e) { + case Event.Scroll: + case Event.Document: + case Event.Resize: + exploded.data = viewport(entry.d, entry.e); + break; case Event.Discover: case Event.Mutation: console.warn("!Event: " + (entry.e === Event.Mutation ? "Mutation" : "Discover") + " | Payload #" + payload.n + " | Event #" + count + " | Nodes #" + entry.d.length + " | First Id: " + (entry.d.length > 0 ? entry.d[0] : -1) + " | Events: " + JSON.stringify(entry.d)); - exploded.data = dom(entry.d); + exploded.data = dom(entry.d, entry.e); break; case Event.Metrics: - exploded.data = metrics(entry.d); + exploded.data = metrics(entry.d, entry.e); break; } decoded.push(exploded); @@ -52,10 +58,26 @@ export function render(payload: IPayload, placeholder: HTMLIFrameElement): void case Event.Mutation: markup(entry.data, placeholder); break; + case Event.Resize: + resize(entry.data, placeholder); + break; } } } +function resize(data: DecodedToken[], placeholder: HTMLIFrameElement): void { + let availableWidth = placeholder.contentWindow.innerWidth; + let width = data[0].width; + let height = data[0].height; + let scale = Math.min(availableWidth / width, 1); + placeholder.style.width = width + "px"; + placeholder.style.height = height + "px"; + placeholder.style.transformOrigin = "0px 0px 0px"; + placeholder.style.transform = "scale(" + scale + ")"; + placeholder.style.border = "1px solid #ccc"; + placeholder.style.overflow = "hidden"; +} + function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { let doc = placeholder.contentDocument; for (let node of data) { @@ -80,7 +102,7 @@ function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { let textElement = element(node.id); textElement = textElement ? textElement : doc.createTextNode(null); textElement.nodeValue = node.value; - insert(node.id, parent, textElement, next); + insert(node, parent, textElement, next); break; case "HTML": let newDoc = doc.implementation.createHTMLDocument(""); @@ -91,13 +113,22 @@ function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { if (doc.body) { doc.body.parentNode.removeChild(doc.body); } nodes[node.id] = doc.documentElement; break; + case "HEAD": + let head = element(node.id); + head = head ? head : doc.createElement(node.tag); + let base = doc.createElement("base"); + base.href = "https://www.bing.com/"; + head.appendChild(base); + setAttributes(head as HTMLElement, node.attributes); + insert(node, parent, head, next); + break; default: let domElement = element(node.id); domElement = domElement ? domElement : doc.createElement(node.tag); if (!node.attributes) { node.attributes = {}; } node.attributes["data-id"] = `${node.id}`; setAttributes(domElement as HTMLElement, node.attributes); - insert(node.id, parent, domElement, next); + insert(node, parent, domElement, next); break; } } @@ -107,9 +138,9 @@ function element(nodeId: number): Node { return nodeId !== null && nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; } -function insert(id: number, parent: Node, node: Node, next: Node): void { +function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void { if (parent !== null) { parent.insertBefore(node, next); } - nodes[id] = node; + nodes[data.id] = node; } function setAttributes(node: HTMLElement, attributes: object): void { diff --git a/decode/dom.ts b/decode/dom.ts index 2f6e92c5..9d8bd716 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -1,8 +1,8 @@ import { resolve } from "../src/data/token"; -import { DecodedToken, Token } from "../types/data"; +import { DecodedToken, Event, Token } from "../types/data"; import { IDecodedNode } from "../types/dom"; -export default function(tokens: Token[]): DecodedToken[] { +export default function(tokens: Token[], event: Event): DecodedToken[] { let number = 0; let lastType = null; let node = []; @@ -44,6 +44,9 @@ export default function(tokens: Token[]): DecodedToken[] { lastType = type; } + // Process last node + decoded.push(process(node, tagIndex)); + return decoded; } diff --git a/decode/metrics.ts b/decode/metrics.ts index 52042c6f..8f64ee3e 100644 --- a/decode/metrics.ts +++ b/decode/metrics.ts @@ -1,7 +1,7 @@ -import { DecodedToken, Token } from "../types/data"; +import { DecodedToken, Event, Token } from "../types/data"; import { Metric } from "../types/metrics"; -export default function(tokens: Token[]): DecodedToken[] { +export default function(tokens: Token[], event: Event): DecodedToken[] { let lastType = null; let metric = []; let decoded: DecodedToken[] = []; diff --git a/decode/viewport.ts b/decode/viewport.ts new file mode 100644 index 00000000..24432bc1 --- /dev/null +++ b/decode/viewport.ts @@ -0,0 +1,19 @@ +import { DecodedToken, Event, Token } from "../types/data"; + +export default function(tokens: Token[], event: Event): DecodedToken[] { + let decoded: DecodedToken[] = []; + switch (event) { + case Event.Resize: + case Event.Document: + decoded.push({ width: tokens[0], height: tokens[1] }); + break; + case Event.Scroll: + let time = 0; + for (let i = 0; i < tokens.length; i = i + 3) { + time += tokens[i] as number; + decoded.push({ time, x: tokens[i + 1], y: tokens[i + 2] }); + } + break; + } + return decoded; +} diff --git a/src/core/config.ts b/src/core/config.ts index d3c333fe..4cdd80ad 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -5,6 +5,7 @@ let config: IConfig = { lookahead: 250, distance: 20, delay: 250, + cssRules: false, tokens: [], upload: null }; diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 37de936c..4592dd0f 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -1,4 +1,4 @@ -import { Action, INodeData, INodeValue, Source } from "@clarity-types/dom"; +import { INodeData, INodeValue, Source } from "@clarity-types/dom"; import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; @@ -26,7 +26,7 @@ export function getId(node: Node, autogen: boolean = true): number { export function add(node: Node, data: INodeData, source: Source): void { let id = getId(node); let parentId = node.parentElement ? getId(node.parentElement) : null; - let nextId = node.nextSibling ? getId(node.nextSibling, false) : null; + let nextId = getNextId(node); if (parentId >= 0 && values[parentId]) { values[parentId].children.push(id); @@ -40,7 +40,7 @@ export function add(node: Node, data: INodeData, source: Source): void { children: [], active: true, update: true, - track: [time(), Action.Add, source], + track: [[time(), source]], data }; } @@ -48,7 +48,7 @@ export function add(node: Node, data: INodeData, source: Source): void { export function update(node: Node, data: INodeData, source: Source): void { let id = getId(node); let parentId = node.parentElement ? getId(node.parentElement) : null; - let nextId = node.nextSibling ? getId(node.nextSibling) : null; + let nextId = getNextId(node); if (id in values) { let value = values[id]; @@ -89,12 +89,19 @@ export function update(node: Node, data: INodeData, source: Source): void { } value["update"] = true; - value["track"].push(time()); - value["track"].push(Action.Update); - value["track"].push(source); + value["track"].push([time(), source]); } } +function getNextId(node: Node): number { + let id = null; + while (id === null && node.nextSibling) { + id = getId(node.nextSibling, false); + node = node.nextSibling; + } + return id; +} + export function getNode(id: number): Node { if (id in nodes) { return nodes[id]; diff --git a/types/core.d.ts b/types/core.d.ts index 5cc7ae4a..6da4705c 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -14,6 +14,7 @@ export interface IConfig { lookahead?: number; distance?: number; delay?: number; + cssRules?: boolean; tokens?: string[]; upload?: (payload: IPayload) => void; } diff --git a/types/dom.d.ts b/types/dom.d.ts index dcd07dc7..6cf93504 100644 --- a/types/dom.d.ts +++ b/types/dom.d.ts @@ -6,12 +6,6 @@ export const enum Source { CharacterData } -export const enum Action { - Add, - Update, - Remove -} - export interface IAttributes { [key: string]: string; } @@ -29,7 +23,7 @@ export interface INodeValue { next: number; children: number[]; data: INodeData; - track: number[]; + track: number[][]; active?: boolean; update?: boolean; } From ba80d1f6f53da58e5f1bed65d0bfb4257e695b01 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 8 Aug 2019 09:53:07 -0700 Subject: [PATCH 027/105] Special handling for style tags --- decode/clarity.ts | 16 +++++++++++----- decode/dom.ts | 4 +++- src/dom/node.ts | 38 +++++++++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 0a3b99a6..429bc1a8 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -114,14 +114,20 @@ function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { nodes[node.id] = doc.documentElement; break; case "HEAD": - let head = element(node.id); - head = head ? head : doc.createElement(node.tag); + let headElement = element(node.id); + headElement = headElement ? headElement : doc.createElement(node.tag); let base = doc.createElement("base"); base.href = "https://www.bing.com/"; - head.appendChild(base); - setAttributes(head as HTMLElement, node.attributes); - insert(node, parent, head, next); + headElement.appendChild(base); + setAttributes(headElement as HTMLElement, node.attributes); + insert(node, parent, headElement, next); break; + case "STYLE": + let styleElement = element(node.id); + styleElement = styleElement ? styleElement : doc.createElement(node.tag); + setAttributes(styleElement as HTMLElement, node.attributes); + styleElement.textContent = node.value; + insert(node, parent, styleElement, next); default: let domElement = element(node.id); domElement = domElement ? domElement : doc.createElement(node.tag); diff --git a/decode/dom.ts b/decode/dom.ts index 9d8bd716..e84c19df 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -66,7 +66,9 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let token = node[i] as string; let keyIndex = token.indexOf("="); let parts = token.split("*"); - if (output.tag !== "*T" && keyIndex > 0) { + if (i === (node.length - 1) && output.tag === "STYLE") { + value = token; + } else if (output.tag !== "*T" && keyIndex > 0) { hasAttribute = true; attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); } else if (parts.length === 4) { diff --git a/src/dom/node.ts b/src/dom/node.ts index 073cc57a..fe8b631c 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -1,4 +1,5 @@ import { Source } from "@clarity-types/dom"; +import config from "@src/core/config"; import * as nodes from "./virtualdom"; let ignoreAttributes = ["title", "alt", "onload", "onfocus"]; @@ -18,8 +19,9 @@ export default function(node: Node, source: Source): void { case Node.TEXT_NODE: // Account for this text node only if we are tracking the parent node // We do not wish to track text nodes for ignored parent nodes, like script tags + // Also, we do not track text nodes for STYLE tags let parent = node.parentElement; - if (parent && nodes.has(parent)) { + if (parent && nodes.has(parent) && parent.tagName !== "STYLE") { let textData = { tag: "*T", value: node.nodeValue }; textData["layout"] = getTextLayout(node); nodes[call](node, textData, source); @@ -27,13 +29,19 @@ export default function(node: Node, source: Source): void { break; case Node.ELEMENT_NODE: let element = (node as HTMLElement); - switch (element.tagName) { + let tag = element.tagName; + switch (tag) { case "SCRIPT": case "NOSCRIPT": case "META": break; + case "STYLE": + let attributes = getAttributes(element.attributes); + let styleData = { tag, attributes, value: getStyleValue(element as HTMLStyleElement) }; + nodes[call](node, styleData, source); + break; default: - let data = { tag: element.tagName, attributes: getAttributes(element.attributes) }; + let data = { tag, attributes: getAttributes(element.attributes) }; data["layout"] = getLayout(element); nodes[call](node, data, source); break; @@ -44,6 +52,30 @@ export default function(node: Node, source: Source): void { } } +function getStyleValue(style: HTMLStyleElement): string { + let value = style.textContent; + if (value.length === 0 || config.cssRules) { + let cssRules = null; + + // Firefox throws a SecurityError when trying to access cssRules of a stylesheet from a different domain + try { + let sheet = style.sheet as CSSStyleSheet; + cssRules = sheet ? sheet.cssRules : []; + } catch (e) { + if (e.name !== "SecurityError") { + throw e; + } + } + + if (cssRules !== null) { + for (let i = 0; i < cssRules.length; i++) { + value += cssRules[i].cssText; + } + } + } + return value; +} + function getAttributes(attributes: NamedNodeMap): {[key: string]: string} { let output = {}; if (attributes && attributes.length > 0) { From e74374811fc0f6792f3f1a1ec9d59dc0848d45c7 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 8 Aug 2019 16:57:00 -0700 Subject: [PATCH 028/105] Handling empty attributes --- decode/clarity.ts | 4 ++-- src/dom/encode.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 429bc1a8..423cd6a8 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -86,7 +86,7 @@ function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { let next = element(node.next); switch (node.tag) { case "*D": - if (typeof XMLSerializer !== "undefined" && false) { + if (typeof XMLSerializer !== "undefined") { doc.open(); doc.write(new XMLSerializer().serializeToString( doc.implementation.createDocumentType( @@ -156,7 +156,7 @@ function setAttributes(node: HTMLElement, attributes: object): void { } } for (let attribute in attributes) { - if (attributes[attribute]) { + if (attributes[attribute] !== undefined) { node.setAttribute(attribute, attributes[attribute]); } } diff --git a/src/dom/encode.ts b/src/dom/encode.ts index 4384f0d7..452ab81e 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -33,7 +33,7 @@ export default async function(timer: Timer): Promise { break; case "attributes": for (let attr in data[key]) { - if (data[key][attr]) { + if (data[key][attr] !== undefined) { metadata.push(`${attr}=${data[key][attr]}`); } } @@ -48,6 +48,9 @@ export default async function(timer: Timer): Promise { break; case "value": let parent = nodes.getNode(value.parent); + if (parent === null) { + console.log("Node data: " + JSON.stringify(data)); + } let parentTag = nodes.get(parent).data.tag; metadata.push(text(parentTag, data[key])); break; From 61a7b0033088fef7898b34f3519c2d4a8dd1f187 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 9 Aug 2019 08:32:01 -0700 Subject: [PATCH 029/105] Adding svg support while decoding --- decode/clarity.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 423cd6a8..9d3170d1 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -6,6 +6,7 @@ import viewport from "./viewport"; let nodes = {}; let pageId: string = null; +let svgns: string = "http://www.w3.org/2000/svg"; let payloads: IPayload[] = []; export function json(payload: IPayload): IDecodedEvent[] { @@ -130,7 +131,7 @@ function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { insert(node, parent, styleElement, next); default: let domElement = element(node.id); - domElement = domElement ? domElement : doc.createElement(node.tag); + domElement = domElement ? domElement : createElement(doc, node.tag, parent as HTMLElement); if (!node.attributes) { node.attributes = {}; } node.attributes["data-id"] = `${node.id}`; setAttributes(domElement as HTMLElement, node.attributes); @@ -140,6 +141,20 @@ function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { } } +function createElement(doc: Document, tag: string, parent: HTMLElement): HTMLElement { + if (tag === "svg") { + return doc.createElementNS(svgns, tag) as HTMLElement; + } else { + while (parent && parent.tagName !== "BODY") { + if (parent.tagName === "svg") { + return doc.createElementNS(svgns, tag) as HTMLElement; + } + parent = parent.parentElement; + } + } + return doc.createElement(tag); +} + function element(nodeId: number): Node { return nodeId !== null && nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; } From 5a11da297c57ffd45d43e3ec2871221a17fdcab3 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 9 Aug 2019 16:50:04 -0700 Subject: [PATCH 030/105] Handling mutations on text node --- decode/clarity.ts | 7 ++++++- src/dom/encode.ts | 2 +- src/dom/node.ts | 4 +++- src/dom/virtualdom.ts | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 9d3170d1..ba00ea65 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -160,7 +160,12 @@ function element(nodeId: number): Node { } function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void { - if (parent !== null) { parent.insertBefore(node, next); } + if (parent !== null) { + next = next && next.parentElement !== parent ? null : next; + parent.insertBefore(node, next); + } else if (parent === null && node.parentElement !== null) { + node.parentElement.removeChild(node); + } nodes[data.id] = node; } diff --git a/src/dom/encode.ts b/src/dom/encode.ts index 452ab81e..72183bea 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -51,7 +51,7 @@ export default async function(timer: Timer): Promise { if (parent === null) { console.log("Node data: " + JSON.stringify(data)); } - let parentTag = nodes.get(parent).data.tag; + let parentTag = nodes.get(parent) ? nodes.get(parent).data.tag : null; metadata.push(text(parentTag, data[key])); break; } diff --git a/src/dom/node.ts b/src/dom/node.ts index fe8b631c..67400761 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -20,8 +20,10 @@ export default function(node: Node, source: Source): void { // Account for this text node only if we are tracking the parent node // We do not wish to track text nodes for ignored parent nodes, like script tags // Also, we do not track text nodes for STYLE tags + // The only exception is when we receive a mutation to remove the text node, in that case + // parent will be null, but we can still process the node by checking it's an update call. let parent = node.parentElement; - if (parent && nodes.has(parent) && parent.tagName !== "STYLE") { + if (call === "update" || (parent && nodes.has(parent) && parent.tagName !== "STYLE")) { let textData = { tag: "*T", value: node.nodeValue }; textData["layout"] = getTextLayout(node); nodes[call](node, textData, source); diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 4592dd0f..0301c8f1 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -46,7 +46,7 @@ export function add(node: Node, data: INodeData, source: Source): void { } export function update(node: Node, data: INodeData, source: Source): void { - let id = getId(node); + let id = getId(node, false); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); From 5d2882899534ec42ea9d18eec27e6e1edb986e94 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sat, 10 Aug 2019 07:11:23 -0700 Subject: [PATCH 031/105] Bug fixes and visualization updates --- decode/clarity.ts | 27 ++++++++++++++++-------- src/dom/virtualdom.ts | 48 ++++++++++++++++++++----------------------- types/dom.d.ts | 9 +++++--- 3 files changed, 46 insertions(+), 38 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index ba00ea65..99fed57e 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -106,12 +106,16 @@ function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { insert(node, parent, textElement, next); break; case "HTML": - let newDoc = doc.implementation.createHTMLDocument(""); - let docElement = newDoc.documentElement; - let pointer = doc.importNode(docElement, true); - doc.replaceChild(pointer, doc.documentElement); - if (doc.head) { doc.head.parentNode.removeChild(doc.head); } - if (doc.body) { doc.body.parentNode.removeChild(doc.body); } + let docElement = element(node.id); + if (!docElement) { + let newDoc = doc.implementation.createHTMLDocument(""); + docElement = newDoc.documentElement; + let pointer = doc.importNode(docElement, true); + doc.replaceChild(pointer, doc.documentElement); + if (doc.head) { doc.head.parentNode.removeChild(doc.head); } + if (doc.body) { doc.body.parentNode.removeChild(doc.body); } + } + setAttributes(docElement as HTMLElement, node.attributes); nodes[node.id] = doc.documentElement; break; case "HEAD": @@ -170,11 +174,16 @@ function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void } function setAttributes(node: HTMLElement, attributes: object): void { - for (let attribute in node.attributes) { - if (node.hasAttribute(attribute)) { - node.removeAttribute(attribute); + // First remove all its existing attributes + if (node.attributes) { + let length = node.attributes.length; + while (node.attributes && length > 0) { + node.removeAttribute(node.attributes[0].name); + length--; } } + + // Add new attributes for (let attribute in attributes) { if (attributes[attribute] !== undefined) { node.setAttribute(attribute, attributes[attribute]); diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 0301c8f1..7c9d4806 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -1,4 +1,4 @@ -import { INodeData, INodeValue, Source } from "@clarity-types/dom"; +import { INodeChange, INodeData, INodeValue, Source } from "@clarity-types/dom"; import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; @@ -6,13 +6,15 @@ let index: number = 1; let nodes: Node[] = []; let values: INodeValue[] = []; +let updates: number[] = []; +let changes: INodeChange[][] = []; let backupIndex: number; let backupNodes: Node[]; -let backupValues: Node[]; +let backupValues: INodeValue[]; // For debugging -window["DOM"] = { getId, get, getNode }; +window["DOM"] = { getId, get, getNode, changes }; export function getId(node: Node, autogen: boolean = true): number { if (node === null) { return null; } @@ -38,11 +40,9 @@ export function add(node: Node, data: INodeData, source: Source): void { parent: parentId, next: nextId, children: [], - active: true, - update: true, - track: [[time(), source]], data }; + track(id, source); } export function update(node: Node, data: INodeData, source: Source): void { @@ -87,9 +87,7 @@ export function update(node: Node, data: INodeData, source: Source): void { value["data"][key] = data[key]; } } - - value["update"] = true; - value["track"].push([time(), source]); + track(id, source); } } @@ -118,11 +116,6 @@ export function has(node: Node): boolean { return getId(node, false) in nodes; } -export function remove(node: Node): void { - let id = getId(node); - del(id); -} - export function getNodes(): Node[] { let n: Node[] = []; for (let id in nodes) { @@ -135,32 +128,35 @@ export function getNodes(): Node[] { export function summarize(): INodeValue[] { let v = []; - for (let id in values) { - if (values[id].update) { - values[id].update = false; + for (let id of updates) { + if (id in values) { v.push(values[id]); } } + updates = []; return v; } export function backup(): void { backupNodes = Array.from(nodes); - backupValues = JSON.parse(JSON.stringify(values)); + backupValues = copy(values); backupIndex = index; } export function rollback(): void { nodes = Array.from(backupNodes); - values = JSON.parse(JSON.stringify(backupValues)); + values = copy(backupValues); index = backupIndex; } -function del(id: number): void { - let children = values[id].children; - for (let i = 0; i < children.length; i++) { - del(children[i]); - } - values[id].active = false; - values[id].update = true; +function copy(input: INodeValue[]): INodeValue[] { + return JSON.parse(JSON.stringify(input)); +} + +function track(id: number, source: Source): void { + if (updates.indexOf(id) === -1) { updates.push(id); } + let value = copy([values[id]])[0]; + let change = { time: time(), source, value }; + if (!(id in changes)) { changes[id] = []; } + changes[id].push(change); } diff --git a/types/dom.d.ts b/types/dom.d.ts index 6cf93504..c1c8e01a 100644 --- a/types/dom.d.ts +++ b/types/dom.d.ts @@ -23,9 +23,12 @@ export interface INodeValue { next: number; children: number[]; data: INodeData; - track: number[][]; - active?: boolean; - update?: boolean; +} + +export interface INodeChange { + time: number; + source: Source; + value: INodeValue; } export interface IDecodedNode { From 0820c48ca134a1f080ee00af50ee41f4a9ed1eaa Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 11 Aug 2019 11:00:11 -0700 Subject: [PATCH 032/105] Refactoring code and adding support for metadata --- decode/clarity.ts | 147 ++--------------------------- decode/dom.ts | 7 +- decode/metadata.ts | 12 +++ decode/render.ts | 144 ++++++++++++++++++++++++++++ src/clarity.ts | 15 ++- src/core/copy.ts | 3 + src/core/recompute.ts | 2 - src/core/version.ts | 2 + src/data/encode.ts | 13 +++ src/data/{track.ts => metadata.ts} | 26 +++-- src/{core => data}/queue.ts | 2 +- src/data/upload.ts | 15 ++- src/dom/discover.ts | 2 +- src/dom/mutation.ts | 2 +- src/dom/node.ts | 6 ++ src/dom/virtualdom.ts | 14 +-- src/interactions/mouse.ts | 2 +- src/metrics/metrics.ts | 7 +- src/viewport/document.ts | 2 +- src/viewport/resize.ts | 2 +- src/viewport/scroll.ts | 2 +- src/viewport/visibility.ts | 2 +- types/data.d.ts | 19 +++- types/index.d.ts | 1 + 24 files changed, 265 insertions(+), 184 deletions(-) create mode 100644 decode/metadata.ts create mode 100644 decode/render.ts create mode 100644 src/core/copy.ts create mode 100644 src/core/version.ts create mode 100644 src/data/encode.ts rename src/data/{track.ts => metadata.ts} (61%) rename src/{core => data}/queue.ts (94%) diff --git a/decode/clarity.ts b/decode/clarity.ts index 99fed57e..84a40bcf 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,28 +1,25 @@ -import { DecodedToken, Event, IDecodedEvent, IEvent, IPayload } from "../types/data"; -import { IDecodedNode } from "../types/dom"; +import { Event, IDecodedEvent, IEvent, IPayload } from "../types/data"; import dom from "./dom"; -import metrics from "./metrics"; +import metadata from "./metadata"; +import { markup, reset, resize } from "./render"; import viewport from "./viewport"; -let nodes = {}; let pageId: string = null; -let svgns: string = "http://www.w3.org/2000/svg"; let payloads: IPayload[] = []; export function json(payload: IPayload): IDecodedEvent[] { if (pageId !== payload.p) { payloads = []; - nodes = {}; pageId = payload.p; + reset(); } let decoded: IDecodedEvent[] = []; let encoded: IEvent[] = JSON.parse(payload.d); payloads.push(payload); - let count = 0; + for (let entry of encoded) { - let exploded = { time: entry.t, event: entry.e, data: entry.d as DecodedToken[] }; - count++; + let exploded: IDecodedEvent = { time: entry.t, event: entry.e, data: null }; switch (entry.e) { case Event.Scroll: case Event.Document: @@ -31,13 +28,10 @@ export function json(payload: IPayload): IDecodedEvent[] { break; case Event.Discover: case Event.Mutation: - console.warn("!Event: " + (entry.e === Event.Mutation ? "Mutation" : "Discover") + - " | Payload #" + payload.n + " | Event #" + count + " | Nodes #" + entry.d.length + - " | First Id: " + (entry.d.length > 0 ? entry.d[0] : -1) + " | Events: " + JSON.stringify(entry.d)); exploded.data = dom(entry.d, entry.e); break; - case Event.Metrics: - exploded.data = metrics(entry.d, entry.e); + case Event.Metadata: + exploded.data = metadata(entry.d, entry.e); break; } decoded.push(exploded); @@ -65,128 +59,3 @@ export function render(payload: IPayload, placeholder: HTMLIFrameElement): void } } } - -function resize(data: DecodedToken[], placeholder: HTMLIFrameElement): void { - let availableWidth = placeholder.contentWindow.innerWidth; - let width = data[0].width; - let height = data[0].height; - let scale = Math.min(availableWidth / width, 1); - placeholder.style.width = width + "px"; - placeholder.style.height = height + "px"; - placeholder.style.transformOrigin = "0px 0px 0px"; - placeholder.style.transform = "scale(" + scale + ")"; - placeholder.style.border = "1px solid #ccc"; - placeholder.style.overflow = "hidden"; -} - -function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { - let doc = placeholder.contentDocument; - for (let node of data) { - console.log("Markup Node: " + JSON.stringify(node)); - let parent = element(node.parent); - let next = element(node.next); - switch (node.tag) { - case "*D": - if (typeof XMLSerializer !== "undefined") { - doc.open(); - doc.write(new XMLSerializer().serializeToString( - doc.implementation.createDocumentType( - node.attributes["name"], - node.attributes["publicId"], - node.attributes["systemId"] - ) - )); - doc.close(); - } - break; - case "*T": - let textElement = element(node.id); - textElement = textElement ? textElement : doc.createTextNode(null); - textElement.nodeValue = node.value; - insert(node, parent, textElement, next); - break; - case "HTML": - let docElement = element(node.id); - if (!docElement) { - let newDoc = doc.implementation.createHTMLDocument(""); - docElement = newDoc.documentElement; - let pointer = doc.importNode(docElement, true); - doc.replaceChild(pointer, doc.documentElement); - if (doc.head) { doc.head.parentNode.removeChild(doc.head); } - if (doc.body) { doc.body.parentNode.removeChild(doc.body); } - } - setAttributes(docElement as HTMLElement, node.attributes); - nodes[node.id] = doc.documentElement; - break; - case "HEAD": - let headElement = element(node.id); - headElement = headElement ? headElement : doc.createElement(node.tag); - let base = doc.createElement("base"); - base.href = "https://www.bing.com/"; - headElement.appendChild(base); - setAttributes(headElement as HTMLElement, node.attributes); - insert(node, parent, headElement, next); - break; - case "STYLE": - let styleElement = element(node.id); - styleElement = styleElement ? styleElement : doc.createElement(node.tag); - setAttributes(styleElement as HTMLElement, node.attributes); - styleElement.textContent = node.value; - insert(node, parent, styleElement, next); - default: - let domElement = element(node.id); - domElement = domElement ? domElement : createElement(doc, node.tag, parent as HTMLElement); - if (!node.attributes) { node.attributes = {}; } - node.attributes["data-id"] = `${node.id}`; - setAttributes(domElement as HTMLElement, node.attributes); - insert(node, parent, domElement, next); - break; - } - } -} - -function createElement(doc: Document, tag: string, parent: HTMLElement): HTMLElement { - if (tag === "svg") { - return doc.createElementNS(svgns, tag) as HTMLElement; - } else { - while (parent && parent.tagName !== "BODY") { - if (parent.tagName === "svg") { - return doc.createElementNS(svgns, tag) as HTMLElement; - } - parent = parent.parentElement; - } - } - return doc.createElement(tag); -} - -function element(nodeId: number): Node { - return nodeId !== null && nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; -} - -function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void { - if (parent !== null) { - next = next && next.parentElement !== parent ? null : next; - parent.insertBefore(node, next); - } else if (parent === null && node.parentElement !== null) { - node.parentElement.removeChild(node); - } - nodes[data.id] = node; -} - -function setAttributes(node: HTMLElement, attributes: object): void { - // First remove all its existing attributes - if (node.attributes) { - let length = node.attributes.length; - while (node.attributes && length > 0) { - node.removeAttribute(node.attributes[0].name); - length--; - } - } - - // Add new attributes - for (let attribute in attributes) { - if (attributes[attribute] !== undefined) { - node.setAttribute(attribute, attributes[attribute]); - } - } -} diff --git a/decode/dom.ts b/decode/dom.ts index e84c19df..e54c4c33 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -1,12 +1,12 @@ import { resolve } from "../src/data/token"; -import { DecodedToken, Event, Token } from "../types/data"; +import { Event, Token } from "../types/data"; import { IDecodedNode } from "../types/dom"; -export default function(tokens: Token[], event: Event): DecodedToken[] { +export default function(tokens: Token[], event: Event): IDecodedNode[] { let number = 0; let lastType = null; let node = []; - let decoded: DecodedToken[] = []; + let decoded: IDecodedNode[] = []; let tagIndex = 0; for (let token of tokens) { let type = typeof(token); @@ -90,6 +90,5 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { if (hasAttribute) { output.attributes = attributes; } if (value) { output.value = value; } - console.log("Node: " + JSON.stringify(output)); return output; } diff --git a/decode/metadata.ts b/decode/metadata.ts new file mode 100644 index 00000000..b3023364 --- /dev/null +++ b/decode/metadata.ts @@ -0,0 +1,12 @@ +import { Event, IMetadata, Token } from "../types/data"; + +export default function(tokens: Token[], event: Event): IMetadata { + return { + version: tokens[0] as string, + pageId: tokens[1] as string, + userId: tokens[2] as string, + siteId: tokens[3] as string, + url: tokens[4] as string, + title: tokens[5] as string + }; +} diff --git a/decode/render.ts b/decode/render.ts new file mode 100644 index 00000000..82ba6536 --- /dev/null +++ b/decode/render.ts @@ -0,0 +1,144 @@ +import { DecodedToken } from "../types/data"; +import { IDecodedNode } from "../types/dom"; + +let nodes = {}; +let svgns: string = "http://www.w3.org/2000/svg"; + +export function reset(): void { + nodes = {}; +} + +export function resize(data: DecodedToken[], placeholder: HTMLIFrameElement): void { + placeholder.removeAttribute("style"); + let margin = 10; + let px = "px"; + let width = data[0].width; + let height = data[0].height; + let availableWidth = (placeholder.contentWindow.innerWidth - (2 * margin)); + let scaleWidth = Math.min(availableWidth / width, 1); + let scaleHeight = Math.min((placeholder.contentWindow.innerHeight - (22 * margin)) / height, 1); + let scale = Math.min(scaleWidth, scaleHeight); + placeholder.style.position = "absolute"; + placeholder.style.width = width + px; + placeholder.style.height = height + px; + placeholder.style.left = ((availableWidth - (width * scale)) / 2) + px; + placeholder.style.transformOrigin = "0 0 0"; + placeholder.style.transform = "scale(" + scale + ")"; + placeholder.style.border = "1px solid #cccccc"; + placeholder.style.margin = margin + px; + placeholder.style.overflow = "hidden"; +} + +export function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { + let doc = placeholder.contentDocument; + for (let node of data) { + let parent = element(node.parent); + let next = element(node.next); + switch (node.tag) { + case "*D": + if (typeof XMLSerializer !== "undefined") { + doc.open(); + doc.write(new XMLSerializer().serializeToString( + doc.implementation.createDocumentType( + node.attributes["name"], + node.attributes["publicId"], + node.attributes["systemId"] + ) + )); + doc.close(); + } + break; + case "*T": + let textElement = element(node.id); + textElement = textElement ? textElement : doc.createTextNode(null); + textElement.nodeValue = node.value; + insert(node, parent, textElement, next); + break; + case "HTML": + let docElement = element(node.id); + if (docElement === null) { + let newDoc = doc.implementation.createHTMLDocument(""); + docElement = newDoc.documentElement; + let pointer = doc.importNode(docElement, true); + doc.replaceChild(pointer, doc.documentElement); + if (doc.head) { doc.head.parentNode.removeChild(doc.head); } + if (doc.body) { doc.body.parentNode.removeChild(doc.body); } + } + setAttributes(docElement as HTMLElement, node.attributes); + nodes[node.id] = doc.documentElement; + break; + case "HEAD": + let headElement = element(node.id); + if (headElement === null) { + headElement = doc.createElement(node.tag); + let base = doc.createElement("base"); + base.href = node.attributes["*B"]; + headElement.appendChild(base); + delete node.attributes["*B"]; + } + setAttributes(headElement as HTMLElement, node.attributes); + insert(node, parent, headElement, next); + break; + case "STYLE": + let styleElement = element(node.id); + styleElement = styleElement ? styleElement : doc.createElement(node.tag); + setAttributes(styleElement as HTMLElement, node.attributes); + styleElement.textContent = node.value; + insert(node, parent, styleElement, next); + default: + let domElement = element(node.id); + domElement = domElement ? domElement : createElement(doc, node.tag, parent as HTMLElement); + if (!node.attributes) { node.attributes = {}; } + node.attributes["data-id"] = `${node.id}`; + setAttributes(domElement as HTMLElement, node.attributes); + insert(node, parent, domElement, next); + break; + } + } +} + +function createElement(doc: Document, tag: string, parent: HTMLElement): HTMLElement { + if (tag === "svg") { + return doc.createElementNS(svgns, tag) as HTMLElement; + } else { + while (parent && parent.tagName !== "BODY") { + if (parent.tagName === "svg") { + return doc.createElementNS(svgns, tag) as HTMLElement; + } + parent = parent.parentElement; + } + } + return doc.createElement(tag); +} + +function element(nodeId: number): Node { + return nodeId !== null && nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; +} + +function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void { + if (parent !== null) { + next = next && next.parentElement !== parent ? null : next; + parent.insertBefore(node, next); + } else if (parent === null && node.parentElement !== null) { + node.parentElement.removeChild(node); + } + nodes[data.id] = node; +} + +function setAttributes(node: HTMLElement, attributes: object): void { + // First remove all its existing attributes + if (node.attributes) { + let length = node.attributes.length; + while (node.attributes && length > 0) { + node.removeAttribute(node.attributes[0].name); + length--; + } + } + + // Add new attributes + for (let attribute in attributes) { + if (attributes[attribute] !== undefined) { + node.setAttribute(attribute, attributes[attribute]); + } + } +} diff --git a/src/clarity.ts b/src/clarity.ts index de12e021..e9411efc 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,7 +1,7 @@ import { IConfig } from "@clarity-types/core"; import config from "@src/core/config"; import * as event from "@src/core/event"; -import * as track from "@src/data/track"; +import * as metadata from "@src/data/metadata"; import * as discover from "@src/dom/discover"; import * as mutation from "@src/dom/mutation"; import * as mouse from "@src/interactions/mouse"; @@ -10,6 +10,8 @@ import * as resize from "@src/viewport/resize"; import * as scroll from "@src/viewport/scroll"; import * as visibility from "@src/viewport/visibility"; +let status = false; + /* Initial discovery of DOM */ export function start(configuration: IConfig = {}): void { @@ -19,7 +21,7 @@ export function start(configuration: IConfig = {}): void { } event.reset(); - track.start(); + metadata.start(); // DOM mutation.start(); @@ -34,10 +36,17 @@ export function start(configuration: IConfig = {}): void { // Pointer mouse.start(); + // Mark Clarity session as active + status = true; } export function end(): void { event.reset(); + metadata.end(); mutation.end(); - track.end(); + status = false; +} + +export function active(): boolean { + return status; } diff --git a/src/core/copy.ts b/src/core/copy.ts new file mode 100644 index 00000000..188c338c --- /dev/null +++ b/src/core/copy.ts @@ -0,0 +1,3 @@ +export default function(input: T): T { + return JSON.parse(JSON.stringify(input)); +} diff --git a/src/core/recompute.ts b/src/core/recompute.ts index 1377e8a9..9c5efd1b 100644 --- a/src/core/recompute.ts +++ b/src/core/recompute.ts @@ -1,7 +1,5 @@ -import * as metrics from "@src/metrics/metrics"; import * as document from "@src/viewport/document"; export default function(): void { document.compute(); - metrics.compute(); } diff --git a/src/core/version.ts b/src/core/version.ts new file mode 100644 index 00000000..77e61c40 --- /dev/null +++ b/src/core/version.ts @@ -0,0 +1,2 @@ +let version = "1.0"; +export default version; diff --git a/src/data/encode.ts b/src/data/encode.ts new file mode 100644 index 00000000..32fd5c96 --- /dev/null +++ b/src/data/encode.ts @@ -0,0 +1,13 @@ +import {Token} from "@clarity-types/data"; +import { metadata } from "@src/data/metadata"; + +export default function(): Token[] { + let tokens = []; + tokens.push(metadata.version); + tokens.push(metadata.pageId); + tokens.push(metadata.userId); + tokens.push(metadata.siteId); + tokens.push(metadata.url); + tokens.push(metadata.title); + return tokens; +} diff --git a/src/data/track.ts b/src/data/metadata.ts similarity index 61% rename from src/data/track.ts rename to src/data/metadata.ts index 8e29df0b..b5cfbb62 100644 --- a/src/data/track.ts +++ b/src/data/metadata.ts @@ -1,21 +1,29 @@ +import { Event, Flush, IMetadata } from "@clarity-types/data"; +import time from "@src/core/time"; +import version from "@src/core/version"; +import encode from "@src/data/encode"; import hash from "@src/data/hash"; +import queue from "@src/data/queue"; -export let pageId: string; -export let sessionId: string; -export let tenantId: string; +export let metadata: IMetadata = null; let count: number = 0; export function start(): void { - pageId = guid(); - sessionId = guid(); - tenantId = hash(top.location.host); count = 0; + metadata = { + version, + pageId: guid(), + userId: guid(), + siteId: hash(location.host), + url: location.href, + title: document.title + }; + + queue(time(), Event.Metadata, encode(), Flush.None); } export function end(): void { - pageId = null; - sessionId = null; - tenantId = null; + metadata = null; count = 0; } diff --git a/src/core/queue.ts b/src/data/queue.ts similarity index 94% rename from src/core/queue.ts rename to src/data/queue.ts index 88dd1812..17578320 100644 --- a/src/core/queue.ts +++ b/src/data/queue.ts @@ -1,6 +1,6 @@ import { Event, Flush, IEvent, Token } from "@clarity-types/data"; import upload from "@src/data/upload"; -import recompute from "./recompute"; +import recompute from "../core/recompute"; let events: IEvent[] = []; let wait = 1000; diff --git a/src/data/upload.ts b/src/data/upload.ts index 50bac306..1ca75ef3 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,14 +1,19 @@ import { IEvent, IPayload } from "@clarity-types/data"; import * as decode from "@decode/clarity"; import config from "@src/core/config"; -import * as track from "@src/data/track"; +import time from "@src/core/time"; +import { metadata, sequence } from "@src/data/metadata"; +import * as metrics from "@src/metrics/metrics"; export default function(events: IEvent[]): void { let payload: IPayload = { - p: track.pageId, - s: track.sessionId, - t: track.tenantId, - n: track.sequence(), + t: time(), + n: sequence(), + v: metadata.version, + p: metadata.pageId, + u: metadata.userId, + s: metadata.siteId, + m: metrics.compute(), d: JSON.stringify(events) }; let upload = config.upload ? config.upload : send; diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 572caf5a..35cd0473 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,9 +1,9 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; import { Timer } from "@clarity-types/metrics"; -import queue from "@src/core/queue"; import * as task from "@src/core/task"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import encode from "@src/dom/encode"; import processNode from "./node"; diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index ba60f002..13b8ad57 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,9 +1,9 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; import { Timer } from "@clarity-types/metrics"; -import queue from "@src/core/queue"; import * as task from "@src/core/task"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import encode from "@src/dom/encode"; import processNode from "./node"; diff --git a/src/dom/node.ts b/src/dom/node.ts index 67400761..dedd6878 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -37,6 +37,12 @@ export default function(node: Node, source: Source): void { case "NOSCRIPT": case "META": break; + case "HEAD": + let head = { tag, attributes: getAttributes(element.attributes) }; + // Capture base href as part of discovering DOM + if (call === "add") { head.attributes["*B"] = location.protocol + "//" + location.hostname; } + nodes[call](node, head, source); + break; case "STYLE": let attributes = getAttributes(element.attributes); let styleData = { tag, attributes, value: getStyleValue(element as HTMLStyleElement) }; diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 7c9d4806..8bcf5c81 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -27,7 +27,7 @@ export function getId(node: Node, autogen: boolean = true): number { export function add(node: Node, data: INodeData, source: Source): void { let id = getId(node); - let parentId = node.parentElement ? getId(node.parentElement) : null; + let parentId = node.parentElement ? getId(node.parentElement, false) : null; let nextId = getNextId(node); if (parentId >= 0 && values[parentId]) { @@ -47,7 +47,7 @@ export function add(node: Node, data: INodeData, source: Source): void { export function update(node: Node, data: INodeData, source: Source): void { let id = getId(node, false); - let parentId = node.parentElement ? getId(node.parentElement) : null; + let parentId = node.parentElement ? getId(node.parentElement, false) : null; let nextId = getNextId(node); if (id in values) { @@ -64,7 +64,7 @@ export function update(node: Node, data: INodeData, source: Source): void { value["parent"] = parentId; // Move this node to the right location under new parent if (parentId !== null && parentId >= 0) { - if (nextId >= 0) { + if (nextId !== null && nextId >= 0) { values[parentId].children.splice(nextId + 1, 0 , id); } else { values[parentId].children.push(id); @@ -75,9 +75,11 @@ export function update(node: Node, data: INodeData, source: Source): void { } // Remove reference to this node from the old parent - let nodeIndex = values[oldParentId].children.indexOf(id); - if (nodeIndex >= 0) { - values[oldParentId].children.splice(nodeIndex, 1); + if (oldParentId !== null && oldParentId >= 0) { + let nodeIndex = values[oldParentId].children.indexOf(id); + if (nodeIndex >= 0) { + values[oldParentId].children.splice(nodeIndex, 1); + } } } diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index eebea43a..993e87d3 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -2,8 +2,8 @@ import { Event } from "@clarity-types/data"; import { IMouseInteraction, Mouse } from "@clarity-types/interactions"; import config from "@src/core/config"; import { bind } from "@src/core/event"; -import queue from "@src/core/queue"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import { getId } from "@src/dom/virtualdom"; import encode from "./encode"; diff --git a/src/metrics/metrics.ts b/src/metrics/metrics.ts index fc2f0806..4c1ca0bb 100644 --- a/src/metrics/metrics.ts +++ b/src/metrics/metrics.ts @@ -1,8 +1,5 @@ -import { Event, Flush } from "@clarity-types/data"; -import queue from "@src/core/queue"; -import time from "@src/core/time"; import encode from "./encode"; -export function compute(): void { - queue(time(), Event.Metrics, encode(), Flush.None); +export function compute(): string { + return JSON.stringify(encode()); } diff --git a/src/viewport/document.ts b/src/viewport/document.ts index c88e830e..d85beebf 100644 --- a/src/viewport/document.ts +++ b/src/viewport/document.ts @@ -1,7 +1,7 @@ import { Event, Flush } from "@clarity-types/data"; import { IDocumentSize } from "@clarity-types/viewport"; -import queue from "@src/core/queue"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import encode from "./encode"; let data: IDocumentSize; diff --git a/src/viewport/resize.ts b/src/viewport/resize.ts index b9b83174..b1abac8e 100644 --- a/src/viewport/resize.ts +++ b/src/viewport/resize.ts @@ -1,8 +1,8 @@ import { Event } from "@clarity-types/data"; import { IResizeViewport } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; -import queue from "@src/core/queue"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import encode from "./encode"; let data: IResizeViewport; diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts index a45562c0..0cb424b5 100644 --- a/src/viewport/scroll.ts +++ b/src/viewport/scroll.ts @@ -1,8 +1,8 @@ import { Event } from "@clarity-types/data"; import { IScrollViewport } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; -import queue from "@src/core/queue"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import encode from "./encode"; let data: IScrollViewport[] = []; diff --git a/src/viewport/visibility.ts b/src/viewport/visibility.ts index 97cfb869..f53e24e2 100644 --- a/src/viewport/visibility.ts +++ b/src/viewport/visibility.ts @@ -1,8 +1,8 @@ import { Event } from "@clarity-types/data"; import { IPageVisibility } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; -import queue from "@src/core/queue"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import encode from "./encode"; let data: IPageVisibility; diff --git a/types/data.d.ts b/types/data.d.ts index 08868e39..8a01bbc1 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -2,6 +2,7 @@ export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); export const enum Event { + Metadata, Discover, Mutation, Metrics, @@ -30,15 +31,27 @@ export interface IEvent { } export interface IPayload { + t: number; + n: number; + v: string; p: string; + u: string; s: string; - t: string; - n: number; + m: string; d: string; } export interface IDecodedEvent { time: number; event: Event; - data: DecodedToken[]; + data: any; +} + +export interface IMetadata { + version: string; + pageId: string; + userId: string; + siteId: string; + url: string; + title: string; } diff --git a/types/index.d.ts b/types/index.d.ts index 778bf7be..0e4947b6 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -4,6 +4,7 @@ interface IClarityJs { version: string; start: (config: IConfig) => void; end: () => void; + active: () => boolean; } declare const clarity: IClarityJs; From ac36d62e6e84332c4cce027a778edb94d71b2acd Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 11 Aug 2019 11:28:05 -0700 Subject: [PATCH 033/105] Removing updated flag from modules --- decode/clarity.ts | 2 +- decode/render.ts | 8 ++++---- decode/viewport.ts | 9 +++++++-- src/interactions/encode.ts | 1 + src/interactions/mouse.ts | 24 ++++++++++++------------ src/metrics/counter.ts | 23 ++++++----------------- src/metrics/encode.ts | 6 +++--- src/metrics/histogram.ts | 8 +++----- src/metrics/mark.ts | 17 +++-------------- src/metrics/timer.ts | 8 +++----- src/viewport/document.ts | 9 ++++----- src/viewport/encode.ts | 14 ++++++++++++-- src/viewport/resize.ts | 9 ++++----- src/viewport/scroll.ts | 25 +++++++++++++------------ src/viewport/visibility.ts | 11 ++++------- types/interactions.d.ts | 1 - types/metrics.d.ts | 36 +++--------------------------------- types/viewport.d.ts | 4 ---- 18 files changed, 83 insertions(+), 132 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 84a40bcf..ac701f9d 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -54,7 +54,7 @@ export function render(payload: IPayload, placeholder: HTMLIFrameElement): void markup(entry.data, placeholder); break; case Event.Resize: - resize(entry.data, placeholder); + resize(entry.data[0], placeholder); break; } } diff --git a/decode/render.ts b/decode/render.ts index 82ba6536..d8a15b22 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ -import { DecodedToken } from "../types/data"; import { IDecodedNode } from "../types/dom"; +import { IResizeViewport } from "../types/viewport"; let nodes = {}; let svgns: string = "http://www.w3.org/2000/svg"; @@ -8,12 +8,12 @@ export function reset(): void { nodes = {}; } -export function resize(data: DecodedToken[], placeholder: HTMLIFrameElement): void { +export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): void { placeholder.removeAttribute("style"); let margin = 10; let px = "px"; - let width = data[0].width; - let height = data[0].height; + let width = data.width; + let height = data.height; let availableWidth = (placeholder.contentWindow.innerWidth - (2 * margin)); let scaleWidth = Math.min(availableWidth / width, 1); let scaleHeight = Math.min((placeholder.contentWindow.innerHeight - (22 * margin)) / height, 1); diff --git a/decode/viewport.ts b/decode/viewport.ts index 24432bc1..fea1f50d 100644 --- a/decode/viewport.ts +++ b/decode/viewport.ts @@ -1,17 +1,22 @@ import { DecodedToken, Event, Token } from "../types/data"; +import { IDocumentSize, IResizeViewport, IScrollViewport } from "../types/viewport"; export default function(tokens: Token[], event: Event): DecodedToken[] { let decoded: DecodedToken[] = []; switch (event) { case Event.Resize: + let r: IResizeViewport = { width: tokens[0] as number, height: tokens[1] as number }; + decoded.push(r); case Event.Document: - decoded.push({ width: tokens[0], height: tokens[1] }); + let d: IDocumentSize = { width: tokens[0] as number, height: tokens[1] as number }; + decoded.push(d); break; case Event.Scroll: let time = 0; for (let i = 0; i < tokens.length; i = i + 3) { time += tokens[i] as number; - decoded.push({ time, x: tokens[i + 1], y: tokens[i + 2] }); + let s: IScrollViewport = { time, x: tokens[i + 1] as number, y: tokens[i + 2] as number }; + decoded.push(s); } break; } diff --git a/src/interactions/encode.ts b/src/interactions/encode.ts index 56bd8426..e13e55ac 100644 --- a/src/interactions/encode.ts +++ b/src/interactions/encode.ts @@ -18,6 +18,7 @@ export default function(type: Event): Token[] { tokens.push(entry.target); if (entry.buttons > 0) { tokens.push(entry.buttons); } } + mouse.reset(); break; } diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index 993e87d3..25e24c25 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -24,7 +24,6 @@ export function start(): void { function handler(type: Mouse, evt: MouseEvent): void { let de = document.documentElement; data.push({ - updated: true, time: time(), type, x: "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null), @@ -40,23 +39,24 @@ function schedule(): void { queue(timestamp, Event.Mouse, encode(Event.Mouse)); } +export function reset(): void { + data = []; +} + export function summarize(): IMouseInteraction[] { let summary: IMouseInteraction[] = []; let index = 0; let last = null; for (let entry of data) { - if (entry.updated) { - let isFirst = index === 0; - if (isFirst - || index === data.length - 1 - || checkDistance(last, entry)) { - timestamp = isFirst ? entry.time : timestamp; - summary.push(entry); - } - index++; - entry.updated = false; - last = entry; + let isFirst = index === 0; + if (isFirst + || index === data.length - 1 + || checkDistance(last, entry)) { + timestamp = isFirst ? entry.time : timestamp; + summary.push(entry); } + index++; + last = entry; } return summary; } diff --git a/src/metrics/counter.ts b/src/metrics/counter.ts index d646f707..5b901051 100644 --- a/src/metrics/counter.ts +++ b/src/metrics/counter.ts @@ -1,22 +1,11 @@ -import { Counter, ICounter, ICounterSummary} from "@clarity-types/metrics"; +import { Counter, ICounter } from "@clarity-types/metrics"; -let tracker: ICounter = {}; -let summary: ICounterSummary = {}; +export let data: ICounter = {}; export function increment(key: Counter, counter: number = 1): void { - if (!(key in tracker)) { - tracker[key] = { updated: true, counter }; + if (!(key in data)) { + data[key] = { updated: true, counter }; } - tracker[key].updated = true; - tracker[key].counter += counter; -} - -export function summarize(): ICounterSummary { - for (let key in tracker) { - if (tracker[key]) { - summary[key] = { counter: tracker[key].counter }; - tracker[key].updated = false; - } - } - return summary; + data[key].updated = true; + data[key].counter += counter; } diff --git a/src/metrics/encode.ts b/src/metrics/encode.ts index d9c129cc..33a08ad8 100644 --- a/src/metrics/encode.ts +++ b/src/metrics/encode.ts @@ -8,12 +8,12 @@ export default function(): Token[] { let metrics = []; // Encode counters - let counters = counter.summarize(); + let counters = counter.data; for (let key in counters) { if (counters[key]) { let c = counters[key]; metrics.push(key); - metrics.push(c.counter); + metrics.push(c); } } @@ -43,7 +43,7 @@ export default function(): Token[] { } // Encode marks - let marks = mark.summarize(); + let marks = mark.data; for (let m of marks) { metrics.push(m.mark); metrics.push(m.start); diff --git a/src/metrics/histogram.ts b/src/metrics/histogram.ts index 4fabb72f..af93bd51 100644 --- a/src/metrics/histogram.ts +++ b/src/metrics/histogram.ts @@ -5,24 +5,22 @@ let summary: IHistogramSummary = {}; export function observe(key: Histogram, value: number): void { if (!(key in tracker)) { - tracker[key] = { updated: true, values: [] }; + tracker[key] = []; } - tracker[key].updated = true; - tracker[key].values.push(value); + tracker[key].push(value); } export function summarize(): IHistogramSummary { for (let key in tracker) { if (tracker[key]) { summary[key] = { sum: 0, min: -1, max: -1, sumsquared: 0, count: 0 }; - for (let value of tracker[key].values) { + for (let value of tracker[key]) { summary[key].sum += value; summary[key].min = summary[key].count > 0 ? Math.min(summary[key].min, value) : value; summary[key].max = summary[key].count > 0 ? Math.max(summary[key].max, value) : value; summary[key].sumsquared += value; summary[key].count++; } - tracker[key].updated = false; } } return summary; diff --git a/src/metrics/mark.ts b/src/metrics/mark.ts index 3008c4f1..7170987e 100644 --- a/src/metrics/mark.ts +++ b/src/metrics/mark.ts @@ -1,19 +1,8 @@ -import {IMark, IMarkSummary, Mark } from "@clarity-types/metrics"; +import {IMark, Mark } from "@clarity-types/metrics"; -let tracker: IMark[] = []; -let summary: IMarkSummary[] = []; +export let data: IMark[] = []; export function mark(key: Mark, tag: string, start: number, end: number = 0): void { end = end > 0 ? end : start; - tracker.push( { updated: true, mark: tag, start, end}); -} - -export function summarize(): IMarkSummary[] { - for (let entry of tracker) { - if (entry) { - summary.push({mark: entry.mark, start: entry.start, end: entry.end }); - entry.updated = false; - } - } - return summary; + data.push( { mark: tag, start, end}); } diff --git a/src/metrics/timer.ts b/src/metrics/timer.ts index b2e7d97a..805df4cb 100644 --- a/src/metrics/timer.ts +++ b/src/metrics/timer.ts @@ -5,21 +5,19 @@ let summary: ITimerSummary = {}; export function observe(key: Timer, value: number): void { if (!(key in tracker)) { - tracker[key] = { updated: true, values: [] }; + tracker[key] = []; } - tracker[key].updated = true; - tracker[key].values.push(value); + tracker[key].push(value); } export function summarize(): ITimerSummary { for (let key in tracker) { if (tracker[key]) { summary[key] = { duration: 0, count: 0 }; - for (let value of tracker[key].values) { + for (let value of tracker[key]) { summary[key].duration += value; summary[key].count++; } - tracker[key].updated = false; } } return summary; diff --git a/src/viewport/document.ts b/src/viewport/document.ts index d85beebf..c642a411 100644 --- a/src/viewport/document.ts +++ b/src/viewport/document.ts @@ -4,7 +4,7 @@ import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "./encode"; -let data: IDocumentSize; +export let data: IDocumentSize; export function start(): void { recompute(); @@ -24,8 +24,7 @@ function recompute(): void { data = { width: body ? body.clientWidth : null, - height: documentHeight, - updated: true + height: documentHeight }; queue(time(), Event.Document, encode(Event.Document), Flush.None); @@ -35,6 +34,6 @@ export function compute(): void { recompute(); } -export function summarize(): IDocumentSize { - return data.updated ? data : null; +export function reset(): void { + data = null; } diff --git a/src/viewport/encode.ts b/src/viewport/encode.ts index e86b7831..0b0dc9c7 100644 --- a/src/viewport/encode.ts +++ b/src/viewport/encode.ts @@ -2,20 +2,24 @@ import {Event, Token} from "@clarity-types/data"; import * as document from "./document"; import * as resize from "./resize"; import * as scroll from "./scroll"; +import * as visibility from "./visibility"; export default function(type: Event): Token[] { let tokens = []; switch (type) { case Event.Resize: - let r = resize.summarize(); + let r = resize.data; tokens.push(r.width); tokens.push(r.height); + resize.reset(); break; case Event.Document: - let d = document.summarize(); + let d = document.data; tokens.push(d.width); tokens.push(d.height); + document.reset(); + break; case Event.Scroll: let s = scroll.summarize(); let timestamp: number = null; @@ -26,6 +30,12 @@ export default function(type: Event): Token[] { tokens.push(entry.x); tokens.push(entry.y); } + scroll.reset(); + break; + case Event.Visibility: + let v = visibility.data; + tokens.push(v.visible); + visibility.reset(); break; } diff --git a/src/viewport/resize.ts b/src/viewport/resize.ts index b1abac8e..db1c5f8e 100644 --- a/src/viewport/resize.ts +++ b/src/viewport/resize.ts @@ -5,7 +5,7 @@ import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "./encode"; -let data: IResizeViewport; +export let data: IResizeViewport; export function start(): void { bind(window, "resize", recompute); @@ -15,12 +15,11 @@ export function start(): void { function recompute(): void { data = { width: "innerWidth" in window ? window.innerWidth : document.documentElement.clientWidth, - height: "innerHeight" in window ? window.innerHeight : document.documentElement.clientHeight, - updated: true + height: "innerHeight" in window ? window.innerHeight : document.documentElement.clientHeight }; queue(time(), Event.Resize, encode(Event.Resize)); } -export function summarize(): IResizeViewport { - return data.updated ? data : null; +export function reset(): void { + data = null; } diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts index 0cb424b5..72100d9e 100644 --- a/src/viewport/scroll.ts +++ b/src/viewport/scroll.ts @@ -19,7 +19,7 @@ export function start(): void { function recompute(): void { let x = "pageXOffset" in window ? window.pageXOffset : document.documentElement.scrollLeft; let y = "pageYOffset" in window ? window.pageYOffset : document.documentElement.scrollTop; - data.push({time: time(), x, y, updated: true}); + data.push({ time: time(), x, y }); if (timeout) { clearTimeout(timeout); } timeout = window.setTimeout(schedule, wait); } @@ -28,23 +28,24 @@ function schedule(): void { queue(timestamp, Event.Scroll, encode(Event.Scroll)); } +export function reset(): void { + data = []; +} + export function summarize(): IScrollViewport[] { let summary: IScrollViewport[] = []; let index = 0; let last = null; for (let entry of data) { - if (entry.updated) { - let isFirst = index === 0; - if (isFirst - || index === data.length - 1 - || checkDistance(last, entry)) { - timestamp = isFirst ? entry.time : timestamp; - summary.push(entry); - } - index++; - entry.updated = false; - last = entry; + let isFirst = index === 0; + if (isFirst + || index === data.length - 1 + || checkDistance(last, entry)) { + timestamp = isFirst ? entry.time : timestamp; + summary.push(entry); } + index++; + last = entry; } return summary; } diff --git a/src/viewport/visibility.ts b/src/viewport/visibility.ts index f53e24e2..96ee347a 100644 --- a/src/viewport/visibility.ts +++ b/src/viewport/visibility.ts @@ -5,7 +5,7 @@ import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "./encode"; -let data: IPageVisibility; +export let data: IPageVisibility; export function start(): void { bind(window, "pagehide", recompute); @@ -15,13 +15,10 @@ export function start(): void { } function recompute(): void { - data = { - visible: "visibilityState" in document ? document.visibilityState : "default", - updated: true - }; + data = { visible: "visibilityState" in document ? document.visibilityState : "default" }; queue(time(), Event.Visibility, encode(Event.Visibility)); } -export function summarize(): IPageVisibility { - return data.updated ? data : null; +export function reset(): void { + data = null; } diff --git a/types/interactions.d.ts b/types/interactions.d.ts index 74158099..25f2cc30 100644 --- a/types/interactions.d.ts +++ b/types/interactions.d.ts @@ -7,7 +7,6 @@ export const enum Mouse { } interface IMouseInteraction { - updated: boolean; time: number; type: Mouse; x: number; diff --git a/types/metrics.d.ts b/types/metrics.d.ts index d486af23..f8c0a9d5 100644 --- a/types/metrics.d.ts +++ b/types/metrics.d.ts @@ -38,30 +38,12 @@ export const enum Mark { // Counter export interface ICounter { - [key: number]: ICounterValue; -} - -interface ICounterValue { - updated: boolean; - counter: number; -} - -export interface ICounterSummary { - [key: string]: ICounterSummaryValue; -} - -interface ICounterSummaryValue { - counter: number; + [key: number]: number; } // Histogram export interface IHistogram { - [key: number]: IHistogramValue; -} - -interface IHistogramValue { - updated: boolean; - values: [number]; + [key: number]: [number]; } export interface IHistogramSummary { @@ -78,13 +60,6 @@ interface IHistogramSummaryValue { // Mark export interface IMark { - mark: string; - updated: boolean; - start: number; - end: number; -} - -export interface IMarkSummary { mark: string; start: number; end: number; @@ -92,12 +67,7 @@ export interface IMarkSummary { // Timer export interface ITimer { - [key: number]: ITimerValue; -} - -interface ITimerValue { - updated: boolean; - values: [number]; + [key: number]: [number]; } export interface ITimerSummary { diff --git a/types/viewport.d.ts b/types/viewport.d.ts index 8b0aca19..fd5f7d1f 100644 --- a/types/viewport.d.ts +++ b/types/viewport.d.ts @@ -1,23 +1,19 @@ export interface IResizeViewport { width: number; height: number; - updated: boolean; } export interface IScrollViewport { time: number; x: number; y: number; - updated: boolean; } export interface IDocumentSize { width: number; height: number; - updated: boolean; } export interface IPageVisibility { visible: string; - updated: boolean; } From 05bc6a81e776420b1625cab5762ffe1704b9304b Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 12 Aug 2019 09:16:24 -0700 Subject: [PATCH 034/105] Refactoring metrics to be uplevel field in payload --- decode/clarity.ts | 43 +++++++++++++----- decode/metrics.ts | 86 ++++++++++++++++-------------------- decode/render.ts | 37 ++++++++++++++-- src/clarity.ts | 3 ++ src/core/task.ts | 14 +++--- src/data/upload.ts | 2 +- src/dom/discover.ts | 4 +- src/dom/encode.ts | 10 ++--- src/dom/mutation.ts | 4 +- src/metrics/counter.ts | 11 ----- src/metrics/encode.ts | 85 +++++++++++++++++++---------------- src/metrics/histogram.ts | 27 ------------ src/metrics/index.ts | 46 +++++++++++++++++++ src/metrics/mark.ts | 8 ---- src/metrics/metric.ts | 24 ++++++++++ src/metrics/metrics.ts | 5 --- src/metrics/timer.ts | 24 ---------- types/data.d.ts | 13 +++++- types/metrics.d.ts | 95 +++++++++++++--------------------------- 19 files changed, 283 insertions(+), 258 deletions(-) delete mode 100644 src/metrics/counter.ts delete mode 100644 src/metrics/histogram.ts create mode 100644 src/metrics/index.ts delete mode 100644 src/metrics/mark.ts create mode 100644 src/metrics/metric.ts delete mode 100644 src/metrics/metrics.ts delete mode 100644 src/metrics/timer.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index ac701f9d..85a00473 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,20 +1,32 @@ -import { Event, IDecodedEvent, IEvent, IPayload } from "../types/data"; +import { Event, IDecodedEvent, IDecodedPayload, IEvent, IPayload } from "../types/data"; import dom from "./dom"; import metadata from "./metadata"; -import { markup, reset, resize } from "./render"; +import metrics from "./metrics"; +import * as r from "./render"; import viewport from "./viewport"; let pageId: string = null; let payloads: IPayload[] = []; -export function json(payload: IPayload): IDecodedEvent[] { +export function json(payload: IPayload): IDecodedPayload { if (pageId !== payload.p) { payloads = []; pageId = payload.p; - reset(); + r.reset(); } - let decoded: IDecodedEvent[] = []; + let decoded: IDecodedPayload = { + time: payload.t, + sequence: payload.n, + version: payload.v, + pageId: payload.p, + userId: payload.u, + siteId: payload.s, + metrics: metrics(JSON.parse(payload.m)), + data: null + }; + + let data: IDecodedEvent[] = []; let encoded: IEvent[] = JSON.parse(payload.d); payloads.push(payload); @@ -34,8 +46,10 @@ export function json(payload: IPayload): IDecodedEvent[] { exploded.data = metadata(entry.d, entry.e); break; } - decoded.push(exploded); + data.push(exploded); } + decoded.data = data; + return decoded; } @@ -45,16 +59,25 @@ export function html(payload: IPayload): string { return placeholder.contentDocument.documentElement.outerHTML; } -export function render(payload: IPayload, placeholder: HTMLIFrameElement): void { +export function render(payload: IPayload, placeholder: HTMLElement): void { let decoded = json(payload); - for (let entry of decoded) { + + let header = placeholder.firstElementChild as HTMLElement; + let iframe = placeholder.lastElementChild as HTMLIFrameElement; + + // Render metrics + r.metrics(decoded.metrics, header); + + // Render events + let events = decoded.data; + for (let entry of events) { switch (entry.event) { case Event.Discover: case Event.Mutation: - markup(entry.data, placeholder); + r.markup(entry.data, iframe); break; case Event.Resize: - resize(entry.data[0], placeholder); + r.resize(entry.data[0], iframe); break; } } diff --git a/decode/metrics.ts b/decode/metrics.ts index 8f64ee3e..171cbc5f 100644 --- a/decode/metrics.ts +++ b/decode/metrics.ts @@ -1,56 +1,44 @@ -import { DecodedToken, Event, Token } from "../types/data"; -import { Metric } from "../types/metrics"; +import Metric from "../src/metrics/metric"; +import { Token } from "../types/data"; +import { IMetric, MetricType } from "../types/metrics"; -export default function(tokens: Token[], event: Event): DecodedToken[] { - let lastType = null; - let metric = []; - let decoded: DecodedToken[] = []; - for (let token of tokens) { - let type = typeof(token); - switch (type) { - case "string": - if (type !== lastType && lastType !== null) { - decoded.push(process(metric)); - metric = []; - } - metric.push(token); +export default function(tokens: Token[]): IMetric { + let metrics: IMetric = { counter: {}, timing: {}, summary: {}, events: [], marks: [] }; + + let i = 0; + let metricType = null; + while (i < tokens.length) { + // Determine metric time for subsequent processing + if (typeof(tokens[i]) === "string") { + metricType = tokens[i++]; + continue; + } + + // Parse metrics + switch (metricType) { + case MetricType.Counter: + metrics.counter[tokens[i++] as Metric] = tokens[i++] as number; + break; + case MetricType.Timing: + metrics.timing[tokens[i++] as Metric] = { duration: tokens[i++] as number, count: tokens[i++] as number }; + break; + case MetricType.Summary: + metrics.summary[tokens[i++] as Metric] = { + sum: tokens[i++] as number, + min: tokens[i++] as number, + max: tokens[i++] as number, + sumsquared: tokens[i++] as number, + count: tokens[i++] as number, + }; + break; + case MetricType.Events: + metrics.events.push({ metric: tokens[i++] as Metric, time: tokens[i++] as number, duration: tokens[i++] as number }); break; - case "number": - metric.push(token); + case MetricType.Marks: + metrics.marks.push({ name: tokens[i++] as string, time: tokens[i++] as number }); break; } - lastType = type; - } - - return decoded; -} - -function process(metric: any[]): DecodedToken { - let name = metric[0]; - let type = name[name.length - 1]; - let output: DecodedToken = { metric: name, type }; - - switch (type) { - case Metric.Timer: - output["duration"] = metric[1]; - output["count"] = metric[2]; - break; - case Metric.Counter: - output["counter"] = metric[1]; - break; - case Metric.Histogram: - output["sum"] = metric[1]; - output["min"] = metric[2]; - output["max"] = metric[3]; - output["sumsquared"] = metric[4]; - output["count"] = metric[5]; - break; - case Metric.Mark: - output["tag"] = metric[1]; - output["start"] = metric[2]; - output["end"] = metric[3]; - break; } - return output; + return metrics; } diff --git a/decode/render.ts b/decode/render.ts index d8a15b22..54e379aa 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,4 +1,6 @@ +import Metric from "../src/metrics/metric"; import { IDecodedNode } from "../types/dom"; +import { IMetric } from "../types/metrics"; import { IResizeViewport } from "../types/viewport"; let nodes = {}; @@ -16,9 +18,9 @@ export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): v let height = data.height; let availableWidth = (placeholder.contentWindow.innerWidth - (2 * margin)); let scaleWidth = Math.min(availableWidth / width, 1); - let scaleHeight = Math.min((placeholder.contentWindow.innerHeight - (22 * margin)) / height, 1); + let scaleHeight = Math.min((placeholder.contentWindow.innerHeight - (16 * margin)) / height, 1); let scale = Math.min(scaleWidth, scaleHeight); - placeholder.style.position = "absolute"; + placeholder.style.position = "relative"; placeholder.style.width = width + px; placeholder.style.height = height + px; placeholder.style.left = ((availableWidth - (width * scale)) / 2) + px; @@ -29,8 +31,35 @@ export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): v placeholder.style.overflow = "hidden"; } -export function markup(data: IDecodedNode[], placeholder: HTMLIFrameElement): void { - let doc = placeholder.contentDocument; +export function metrics(data: IMetric, header: HTMLElement): void { + let html = []; + + // Counters + for (let metric in data.counter) { + if (data.counter[metric]) { + html.push(`

  • ${data.counter[metric]}

    ${Metric[metric]}
    • `); + } + } + + // Timing + for (let metric in data.timing) { + if (data.timing[metric]) { + html.push(`

  • ${data.timing[metric].duration}

    ${Metric[metric]}
    • `); + } + } + + // Summary + for (let metric in data.summary) { + if (data.summary[metric]) { + html.push(`

  • ${data.summary[metric].max}

    ${Metric[metric]}
    • `); + } + } + + header.innerHTML = `
      ${html.join()}
    `; +} + +export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { + let doc = iframe.contentDocument; for (let node of data) { let parent = element(node.parent); let next = element(node.next); diff --git a/src/clarity.ts b/src/clarity.ts index e9411efc..d540c545 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -5,6 +5,7 @@ import * as metadata from "@src/data/metadata"; import * as discover from "@src/dom/discover"; import * as mutation from "@src/dom/mutation"; import * as mouse from "@src/interactions/mouse"; +import * as metrics from "@src/metrics"; import * as document from "@src/viewport/document"; import * as resize from "@src/viewport/resize"; import * as scroll from "@src/viewport/scroll"; @@ -22,6 +23,7 @@ export function start(configuration: IConfig = {}): void { event.reset(); metadata.start(); + metrics.start(); // DOM mutation.start(); @@ -43,6 +45,7 @@ export function start(configuration: IConfig = {}): void { export function end(): void { event.reset(); metadata.end(); + metrics.end(); mutation.end(); status = false; } diff --git a/src/core/task.ts b/src/core/task.ts index afd53b50..d6c704a4 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,30 +1,30 @@ import {ITask } from "@clarity-types/core"; -import { Timer } from "@clarity-types/metrics"; import config from "@src/core/config"; -import * as timer from "@src/metrics/timer"; +import * as metrics from "@src/metrics"; +import Metric from "@src/metrics/metric"; let tracker: ITask = {}; let threshold = config.longtask; -export function longtask(method: Timer): boolean { +export function longtask(method: Metric): boolean { let elapsed = Date.now() - tracker[method]; return (elapsed > threshold); } -export function start(method: Timer): void { +export function start(method: Metric): void { if (!(method in tracker)) { tracker[method] = 0; } tracker[method] = Date.now(); } -export function stop(method: Timer): void { +export function stop(method: Metric): void { let end = Date.now(); let duration = end - tracker[method]; - timer.observe(method, duration); + metrics.timing(method, duration); } -export async function idle(method: Timer): Promise { +export async function idle(method: Metric): Promise { stop(method); await wait(); start(method); diff --git a/src/data/upload.ts b/src/data/upload.ts index 1ca75ef3..a80f52a9 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -3,7 +3,7 @@ import * as decode from "@decode/clarity"; import config from "@src/core/config"; import time from "@src/core/time"; import { metadata, sequence } from "@src/data/metadata"; -import * as metrics from "@src/metrics/metrics"; +import * as metrics from "@src/metrics"; export default function(events: IEvent[]): void { let payload: IPayload = { diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 35cd0473..88443edd 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,10 +1,10 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; -import { Timer } from "@clarity-types/metrics"; import * as task from "@src/core/task"; import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "@src/dom/encode"; +import Metric from "@src/metrics/metric"; import processNode from "./node"; export function start(): void { @@ -14,7 +14,7 @@ export function start(): void { } async function discover(): Promise { - let timer = Timer.Discover; + let timer = Metric.DiscoverTime; task.start(timer); let walker = document.createTreeWalker(document, NodeFilter.SHOW_ALL, null, false); let node = walker.nextNode(); diff --git a/src/dom/encode.ts b/src/dom/encode.ts index 72183bea..ed1b2b54 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -1,17 +1,17 @@ import {Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; -import { Counter, Timer } from "@clarity-types/metrics"; import * as task from "@src/core/task"; import hash from "@src/data/hash"; import {check} from "@src/data/token"; -import * as counter from "@src/metrics/counter"; +import * as metrics from "@src/metrics"; +import Metric from "@src/metrics/metric"; import * as nodes from "./virtualdom"; window["HASH"] = hash; let reference: number = 0; -export default async function(timer: Timer): Promise { +export default async function(timer: Metric): Promise { let markup = []; let values = nodes.summarize(); reference = 0; @@ -25,7 +25,7 @@ export default async function(timer: Timer): Promise { if (data[key]) { switch (key) { case "tag": - counter.increment(Counter.Nodes); + metrics.counter(Metric.Nodes); markup.push(number(value.id)); if (value.parent) { markup.push(number(value.parent)); } if (value.next) { markup.push(number(value.next)); } @@ -49,7 +49,7 @@ export default async function(timer: Timer): Promise { case "value": let parent = nodes.getNode(value.parent); if (parent === null) { - console.log("Node data: " + JSON.stringify(data)); + console.warn("Unexpected | Node data: " + JSON.stringify(data)); } let parentTag = nodes.get(parent) ? nodes.get(parent).data.tag : null; metadata.push(text(parentTag, data[key])); diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 13b8ad57..9f523b70 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,10 +1,10 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; -import { Timer } from "@clarity-types/metrics"; import * as task from "@src/core/task"; import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "@src/dom/encode"; +import Metric from "@src/metrics/metric"; import processNode from "./node"; let observer: MutationObserver; @@ -33,7 +33,7 @@ function handle(mutations: MutationRecord[]): void { } async function process(mutations: MutationRecord[]): Promise { - let timer = Timer.Mutation; + let timer = Metric.MutationTime; task.start(timer); let length = mutations.length; for (let i = 0; i < length; i++) { diff --git a/src/metrics/counter.ts b/src/metrics/counter.ts deleted file mode 100644 index 5b901051..00000000 --- a/src/metrics/counter.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Counter, ICounter } from "@clarity-types/metrics"; - -export let data: ICounter = {}; - -export function increment(key: Counter, counter: number = 1): void { - if (!(key in data)) { - data[key] = { updated: true, counter }; - } - data[key].updated = true; - data[key].counter += counter; -} diff --git a/src/metrics/encode.ts b/src/metrics/encode.ts index 33a08ad8..89cc63f5 100644 --- a/src/metrics/encode.ts +++ b/src/metrics/encode.ts @@ -1,54 +1,63 @@ import {Token} from "@clarity-types/data"; -import * as counter from "./counter"; -import * as histogram from "./histogram"; -import * as mark from "./mark"; -import * as timer from "./timer"; +import { MetricType } from "@clarity-types/metrics"; +import { metrics } from "@src/metrics"; export default function(): Token[] { - let metrics = []; + let output = []; - // Encode counters - let counters = counter.data; - for (let key in counters) { - if (counters[key]) { - let c = counters[key]; - metrics.push(key); - metrics.push(c); + // Encode counter metrics + output.push(MetricType.Counter); + let counters = metrics.counter; + for (let metric in counters) { + if (counters[metric]) { + output.push(parseInt(metric, 10)); + output.push(counters[metric]); } } - // Encode histograms - let histograms = histogram.summarize(); - for (let key in histograms) { - if (histograms[key]) { - let h = histograms[key]; - metrics.push(key); - metrics.push(h.sum); - metrics.push(h.min); - metrics.push(h.max); - metrics.push(h.sumsquared); - metrics.push(h.count); + // Encode summary metrics + output.push(MetricType.Summary); + let summaries = metrics.summary; + for (let metric in summaries) { + if (summaries[metric]) { + let h = summaries[metric]; + output.push(parseInt(metric, 10)); + output.push(h.sum); + output.push(h.min); + output.push(h.max); + output.push(h.sumsquared); + output.push(h.count); } } - // Encode timers - let timers = timer.summarize(); - for (let key in timers) { - if (timers[key]) { - let t = timers[key]; - metrics.push(key); - metrics.push(t.duration); - metrics.push(t.count); + // Encode timing metrics + output.push(MetricType.Timing); + let timings = metrics.timing; + for (let metric in timings) { + if (timings[metric]) { + let l = timings[metric]; + output.push(parseInt(metric, 10)); + output.push(l.duration); + output.push(l.count); } } - // Encode marks - let marks = mark.data; - for (let m of marks) { - metrics.push(m.mark); - metrics.push(m.start); - metrics.push(m.end); + // Encode semantic events + if (metrics.events.length > 0) { output.push(MetricType.Events); } + let events = metrics.events; + for (let event of events) { + output.push(event.metric); + output.push(event.time); + output.push(event.duration); } - return metrics; + // Encode user specified marks + if (metrics.marks.length > 0) { output.push(MetricType.Marks); } + let marks = metrics.marks; + for (let mark of marks) { + output.push(mark.name); + output.push(mark.time); + } + + return output; } diff --git a/src/metrics/histogram.ts b/src/metrics/histogram.ts deleted file mode 100644 index af93bd51..00000000 --- a/src/metrics/histogram.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Histogram, IHistogram, IHistogramSummary} from "@clarity-types/metrics"; - -let tracker: IHistogram = {}; -let summary: IHistogramSummary = {}; - -export function observe(key: Histogram, value: number): void { - if (!(key in tracker)) { - tracker[key] = []; - } - tracker[key].push(value); -} - -export function summarize(): IHistogramSummary { - for (let key in tracker) { - if (tracker[key]) { - summary[key] = { sum: 0, min: -1, max: -1, sumsquared: 0, count: 0 }; - for (let value of tracker[key]) { - summary[key].sum += value; - summary[key].min = summary[key].count > 0 ? Math.min(summary[key].min, value) : value; - summary[key].max = summary[key].count > 0 ? Math.max(summary[key].max, value) : value; - summary[key].sumsquared += value; - summary[key].count++; - } - } - } - return summary; -} diff --git a/src/metrics/index.ts b/src/metrics/index.ts new file mode 100644 index 00000000..ed61b640 --- /dev/null +++ b/src/metrics/index.ts @@ -0,0 +1,46 @@ +import { IMetric } from "@clarity-types/metrics"; +import time from "@src/core/time"; +import Metric from "@src/metrics/metric"; +import encode from "./encode"; + +export let metrics: IMetric = null; + +export function start(): void { + metrics = { counter: {}, timing: {}, summary: {}, events: [], marks: [] }; +} + +export function end(): void { + metrics = null; +} + +export function compute(): string { + return JSON.stringify(encode()); +} + +export function counter(metric: Metric, increment: number = 1): void { + if (!(metric in metrics.counter)) { metrics.counter[metric] = 0; } + metrics.counter[metric] += increment; +} + +export function timing(metric: Metric, duration: number): void { + if (!(metric in metrics.timing)) { metrics.timing[metric] = { duration: 0, count: 0 }; } + metrics.timing[metric].duration += duration; + metrics.timing[metric].count++; +} + +export function summary(metric: Metric, value: number): void { + if (!(metric in metrics.summary)) { metrics.summary[metric] = { sum: 0, min: null, max: null, sumsquared: 0, count: 0 }; } + metrics.summary[metric].sum += value; + metrics.summary[metric].min = metrics.summary[metric].min !== null ? Math.min(metrics.summary[metric].min, value) : value; + metrics.summary[metric].max = metrics.summary[metric].max !== null ? Math.max(metrics.summary[metric].max, value) : value; + metrics.summary[metric].sumsquared += value; + metrics.summary[metric].count++; +} + +export function event(metric: Metric, begin: number, duration: number = 0): void { + metrics.events.push({ metric, time: begin, duration }); +} + +export function mark(name: string): void { + metrics.marks.push({ name, time: time() }); +} diff --git a/src/metrics/mark.ts b/src/metrics/mark.ts deleted file mode 100644 index 7170987e..00000000 --- a/src/metrics/mark.ts +++ /dev/null @@ -1,8 +0,0 @@ -import {IMark, Mark } from "@clarity-types/metrics"; - -export let data: IMark[] = []; - -export function mark(key: Mark, tag: string, start: number, end: number = 0): void { - end = end > 0 ? end : start; - data.push( { mark: tag, start, end}); -} diff --git a/src/metrics/metric.ts b/src/metrics/metric.ts new file mode 100644 index 00000000..738a1045 --- /dev/null +++ b/src/metrics/metric.ts @@ -0,0 +1,24 @@ +enum Metric { + /* Timing */ + DiscoverTime, + MutationTime, + WireupDelay, + ActiveTime, + /* Counter */ + Nodes, + Bytes, + Mutations, + Interactions, + Clicks, + Errors, + /* Summary */ + ViewportWidth, + ViewportHeight, + DocumentWidth, + DocumentHeight, + /* Semantic Events */ + Click, + Interaction +} + +export default Metric; diff --git a/src/metrics/metrics.ts b/src/metrics/metrics.ts deleted file mode 100644 index 4c1ca0bb..00000000 --- a/src/metrics/metrics.ts +++ /dev/null @@ -1,5 +0,0 @@ -import encode from "./encode"; - -export function compute(): string { - return JSON.stringify(encode()); -} diff --git a/src/metrics/timer.ts b/src/metrics/timer.ts deleted file mode 100644 index 805df4cb..00000000 --- a/src/metrics/timer.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { ITimer, ITimerSummary, Timer } from "@clarity-types/metrics"; - -let tracker: ITimer = {}; -let summary: ITimerSummary = {}; - -export function observe(key: Timer, value: number): void { - if (!(key in tracker)) { - tracker[key] = []; - } - tracker[key].push(value); -} - -export function summarize(): ITimerSummary { - for (let key in tracker) { - if (tracker[key]) { - summary[key] = { duration: 0, count: 0 }; - for (let value of tracker[key]) { - summary[key].duration += value; - summary[key].count++; - } - } - } - return summary; -} diff --git a/types/data.d.ts b/types/data.d.ts index 8a01bbc1..a2d98ff7 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,3 +1,5 @@ +import { IMetric } from "./metrics"; + export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); @@ -5,7 +7,6 @@ export const enum Event { Metadata, Discover, Mutation, - Metrics, Mouse, Touch, Keyboard, @@ -41,6 +42,16 @@ export interface IPayload { d: string; } +export interface IDecodedPayload { + time: number; + sequence: number; + version: string; + pageId: string; + userId: string; + siteId: string; + metrics: IMetric; + data: IDecodedEvent[]; +} export interface IDecodedEvent { time: number; event: Event; diff --git a/types/metrics.d.ts b/types/metrics.d.ts index f8c0a9d5..f236b487 100644 --- a/types/metrics.d.ts +++ b/types/metrics.d.ts @@ -1,80 +1,47 @@ -// Metric Names -export const enum Metric { - Timer = "t", - Counter = "c", - Histogram = "h", - Mark = "m" +export const enum MetricType { + Counter = "C", + Timing = "T", + Summary = "S", + Events = "E", + Marks = "M" } -export const enum Timer { - Discover = "dt", - Mutation = "mt", - Wireup = "wt", - Active = "at" +export interface IMetric { + timing: ITiming; + counter: ICounter; + summary: ISummary; + events: ISemanticEvent[]; + marks: IMark[]; } -export const enum Counter { - Nodes = "nc", - Bytes = "bc", - Mutations = "mc", - Swipes = "sc", +export interface ITiming { + [key: number]: { + duration: number; + count: number; + }; } -export const enum Histogram { - PointerDistance = "ph", - ViewportX = "xh", - ViewportY = "yh", - ViewportWidth = "wh", - ViewportHeight = "hh", - DocumentWidth = "dwh", - DocumentHeight = "dhh" -} - -export const enum Mark { - Click = "cm", - Error = "em", - Interaction = "im" -} - -// Counter export interface ICounter { [key: number]: number; } -// Histogram -export interface IHistogram { - [key: number]: [number]; +export interface ISummary { + [key: number]: { + sum: number; + min: number; + max: number; + count: number; + sumsquared: number; + }; } -export interface IHistogramSummary { - [key: string]: IHistogramSummaryValue; -} - -interface IHistogramSummaryValue { - sum: number; - min: number; - max: number; - count: number; - sumsquared: number; +export interface ISemanticEvent { + metric: number; + time: number; + duration: number; } -// Mark export interface IMark { - mark: string; - start: number; - end: number; -} - -// Timer -export interface ITimer { - [key: number]: [number]; -} - -export interface ITimerSummary { - [key: number]: ITimerSummaryValue; -} - -interface ITimerSummaryValue { - duration: number; - count: number; + name: string; + time: number; } From 9007b7832e9d62a98fc9e6b5bf7ccca70e481ed7 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 12 Aug 2019 19:36:19 -0700 Subject: [PATCH 035/105] Reogranizing envelope --- decode/clarity.ts | 48 +++++++++++++++++++------------------------- decode/envelope.ts | 11 ++++++++++ decode/metadata.ts | 13 ++++++------ decode/render.ts | 12 +++++++---- src/data/encode.ts | 10 +++++---- src/data/metadata.ts | 13 ++++++------ src/data/upload.ts | 22 ++++++++------------ src/metrics/index.ts | 5 ----- types/core.d.ts | 2 +- types/data.d.ts | 27 ++++++++++--------------- 10 files changed, 79 insertions(+), 84 deletions(-) create mode 100644 decode/envelope.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 85a00473..164b1658 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,5 +1,6 @@ import { Event, IDecodedEvent, IDecodedPayload, IEvent, IPayload } from "../types/data"; import dom from "./dom"; +import envelope from "./envelope"; import metadata from "./metadata"; import metrics from "./metrics"; import * as r from "./render"; @@ -8,26 +9,22 @@ import viewport from "./viewport"; let pageId: string = null; let payloads: IPayload[] = []; -export function json(payload: IPayload): IDecodedPayload { - if (pageId !== payload.p) { - payloads = []; - pageId = payload.p; - r.reset(); - } - +export function json(data: string): IDecodedPayload { + let payload = JSON.parse(data); let decoded: IDecodedPayload = { - time: payload.t, - sequence: payload.n, - version: payload.v, - pageId: payload.p, - userId: payload.u, - siteId: payload.s, - metrics: metrics(JSON.parse(payload.m)), + envelope: envelope(payload.e), + metrics: metrics(payload.m), data: null }; - let data: IDecodedEvent[] = []; - let encoded: IEvent[] = JSON.parse(payload.d); + if (pageId !== decoded.envelope.pageId) { + payloads = []; + pageId = decoded.envelope.pageId; + r.reset(); + } + + let events: IDecodedEvent[] = []; + let encoded: IEvent[] = payload.d; payloads.push(payload); for (let entry of encoded) { @@ -46,24 +43,21 @@ export function json(payload: IPayload): IDecodedPayload { exploded.data = metadata(entry.d, entry.e); break; } - data.push(exploded); + events.push(exploded); } - decoded.data = data; + decoded.data = events; return decoded; } -export function html(payload: IPayload): string { - let placeholder = document.createElement("iframe"); - render(payload, placeholder); - return placeholder.contentDocument.documentElement.outerHTML; +export function html(data: string): string { + let iframe = document.createElement("iframe"); + render(data, iframe); + return iframe.contentDocument.documentElement.outerHTML; } -export function render(payload: IPayload, placeholder: HTMLElement): void { - let decoded = json(payload); - - let header = placeholder.firstElementChild as HTMLElement; - let iframe = placeholder.lastElementChild as HTMLIFrameElement; +export function render(data: string, iframe: HTMLIFrameElement, header?: HTMLElement): void { + let decoded = json(data); // Render metrics r.metrics(decoded.metrics, header); diff --git a/decode/envelope.ts b/decode/envelope.ts new file mode 100644 index 00000000..9579a077 --- /dev/null +++ b/decode/envelope.ts @@ -0,0 +1,11 @@ +import { IEnvelope, Token } from "../types/data"; + +export default function(tokens: Token[]): IEnvelope { + return { + sequence: tokens[0] as number, + version: tokens[1] as string, + pageId: tokens[2] as string, + userId: tokens[3] as string, + projectId: tokens[4] as string + }; +} diff --git a/decode/metadata.ts b/decode/metadata.ts index b3023364..2d6ce94f 100644 --- a/decode/metadata.ts +++ b/decode/metadata.ts @@ -2,11 +2,12 @@ import { Event, IMetadata, Token } from "../types/data"; export default function(tokens: Token[], event: Event): IMetadata { return { - version: tokens[0] as string, - pageId: tokens[1] as string, - userId: tokens[2] as string, - siteId: tokens[3] as string, - url: tokens[4] as string, - title: tokens[5] as string + sequence: tokens[0] as number, + version: tokens[1] as string, + pageId: tokens[2] as string, + userId: tokens[3] as string, + projectId: tokens[4] as string, + url: tokens[5] as string, + title: tokens[6] as string }; } diff --git a/decode/render.ts b/decode/render.ts index 54e379aa..b4ef9da9 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -37,25 +37,29 @@ export function metrics(data: IMetric, header: HTMLElement): void { // Counters for (let metric in data.counter) { if (data.counter[metric]) { - html.push(`

  • ${data.counter[metric]}

    ${Metric[metric]}
    • `); + html.push(`

  • ${data.counter[metric]}

    ${getMetricName(parseInt(metric, 10))}
    • `); } } // Timing for (let metric in data.timing) { if (data.timing[metric]) { - html.push(`

  • ${data.timing[metric].duration}

    ${Metric[metric]}
    • `); + html.push(`

  • ${data.timing[metric].duration}

    ${getMetricName(parseInt(metric, 10))}
    • `); } } // Summary for (let metric in data.summary) { if (data.summary[metric]) { - html.push(`

  • ${data.summary[metric].max}

    ${Metric[metric]}
    • `); + html.push(`

  • ${data.summary[metric].max}

    ${getMetricName(parseInt(metric, 10))}
    • `); } } - header.innerHTML = `
      ${html.join()}
    `; + header.innerHTML = `
      ${html.join("")}
    `; +} + +function getMetricName(metric: Metric): string { + return Metric[metric].replace(/([A-Z])/g, " $1").replace(/^./, function(str: string): string { return str.toUpperCase(); }); } export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { diff --git a/src/data/encode.ts b/src/data/encode.ts index 32fd5c96..d482a363 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -1,13 +1,15 @@ import {Token} from "@clarity-types/data"; import { metadata } from "@src/data/metadata"; -export default function(): Token[] { +export default function(envelope: boolean = false): Token[] { let tokens = []; tokens.push(metadata.version); tokens.push(metadata.pageId); tokens.push(metadata.userId); - tokens.push(metadata.siteId); - tokens.push(metadata.url); - tokens.push(metadata.title); + tokens.push(metadata.projectId); + if (envelope === false) { + tokens.push(metadata.url); + tokens.push(metadata.title); + } return tokens; } diff --git a/src/data/metadata.ts b/src/data/metadata.ts index b5cfbb62..08992928 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,4 +1,4 @@ -import { Event, Flush, IMetadata } from "@clarity-types/data"; +import { Event, Flush, IMetadata, Token } from "@clarity-types/data"; import time from "@src/core/time"; import version from "@src/core/version"; import encode from "@src/data/encode"; @@ -6,15 +6,14 @@ import hash from "@src/data/hash"; import queue from "@src/data/queue"; export let metadata: IMetadata = null; -let count: number = 0; export function start(): void { - count = 0; metadata = { + sequence: 0, version, pageId: guid(), userId: guid(), - siteId: hash(location.host), + projectId: hash(location.host), url: location.href, title: document.title }; @@ -24,11 +23,11 @@ export function start(): void { export function end(): void { metadata = null; - count = 0; } -export function sequence(): number { - return count++; +export function envelope(): Token[] { + metadata.sequence++; + return encode(true); } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript diff --git a/src/data/upload.ts b/src/data/upload.ts index a80f52a9..a20e8289 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,25 +1,19 @@ import { IEvent, IPayload } from "@clarity-types/data"; import * as decode from "@decode/clarity"; import config from "@src/core/config"; -import time from "@src/core/time"; -import { metadata, sequence } from "@src/data/metadata"; -import * as metrics from "@src/metrics"; +import {envelope} from "@src/data/metadata"; +import metrics from "@src/metrics/encode"; export default function(events: IEvent[]): void { let payload: IPayload = { - t: time(), - n: sequence(), - v: metadata.version, - p: metadata.pageId, - u: metadata.userId, - s: metadata.siteId, - m: metrics.compute(), - d: JSON.stringify(events) + e: envelope(), + m: metrics(), + d: events }; let upload = config.upload ? config.upload : send; - upload(payload); + upload(JSON.stringify(payload)); } -function send(payload: IPayload): void { - decode.json(payload); +function send(data: string): void { + decode.json(data); } diff --git a/src/metrics/index.ts b/src/metrics/index.ts index ed61b640..2eaafec9 100644 --- a/src/metrics/index.ts +++ b/src/metrics/index.ts @@ -1,7 +1,6 @@ import { IMetric } from "@clarity-types/metrics"; import time from "@src/core/time"; import Metric from "@src/metrics/metric"; -import encode from "./encode"; export let metrics: IMetric = null; @@ -13,10 +12,6 @@ export function end(): void { metrics = null; } -export function compute(): string { - return JSON.stringify(encode()); -} - export function counter(metric: Metric, increment: number = 1): void { if (!(metric in metrics.counter)) { metrics.counter[metric] = 0; } metrics.counter[metric] += increment; diff --git a/types/core.d.ts b/types/core.d.ts index 6da4705c..0b68d3a9 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -16,7 +16,7 @@ export interface IConfig { delay?: number; cssRules?: boolean; tokens?: string[]; - upload?: (payload: IPayload) => void; + upload?: (data: string) => void; } // Task diff --git a/types/data.d.ts b/types/data.d.ts index a2d98ff7..cc8689c1 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -32,37 +32,32 @@ export interface IEvent { } export interface IPayload { - t: number; - n: number; - v: string; - p: string; - u: string; - s: string; - m: string; - d: string; + e: Token[]; + m: Token[]; + d: IEvent[]; } export interface IDecodedPayload { - time: number; - sequence: number; - version: string; - pageId: string; - userId: string; - siteId: string; + envelope: IEnvelope; metrics: IMetric; data: IDecodedEvent[]; } + export interface IDecodedEvent { time: number; event: Event; data: any; } -export interface IMetadata { +export interface IEnvelope { + sequence: number; version: string; pageId: string; userId: string; - siteId: string; + projectId: string; +} + +export interface IMetadata extends IEnvelope { url: string; title: string; } From 889fd47f3dbc23cbc264251d74425e126481805b Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 13 Aug 2019 07:24:10 -0700 Subject: [PATCH 036/105] Simplifying metrics --- decode/metrics.ts | 33 ++++++++++++++------ decode/render.ts | 70 +++++++++++++++++++------------------------ src/core/task.ts | 4 +-- src/dom/discover.ts | 3 +- src/dom/encode.ts | 4 +-- src/dom/mutation.ts | 2 +- src/metrics/encode.ts | 16 ++-------- src/metrics/index.ts | 29 +++++++----------- src/metrics/metric.ts | 24 --------------- types/data.d.ts | 4 +-- types/metrics.d.ts | 43 ++++++++++++++++++++------ 11 files changed, 111 insertions(+), 121 deletions(-) delete mode 100644 src/metrics/metric.ts diff --git a/decode/metrics.ts b/decode/metrics.ts index 171cbc5f..2aae4d7b 100644 --- a/decode/metrics.ts +++ b/decode/metrics.ts @@ -1,9 +1,27 @@ -import Metric from "../src/metrics/metric"; import { Token } from "../types/data"; -import { IMetric, MetricType } from "../types/metrics"; +import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metrics"; -export default function(tokens: Token[]): IMetric { - let metrics: IMetric = { counter: {}, timing: {}, summary: {}, events: [], marks: [] }; +let map: IMetricMap = {}; + +map[Metric.NodeCount] = { name: "Node Count", unit: ""}; +map[Metric.ByteCount] = { name: "Byte Count", unit: ""}; +map[Metric.MutationCount] = { name: "Mutation Count", unit: ""}; +map[Metric.InteractionCount] = { name: "Interaction Count", unit: ""}; +map[Metric.ClickCount] = { name: "Click Count", unit: ""}; +map[Metric.ErrorCount] = { name: "Error Count", unit: ""}; +map[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; +map[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; +map[Metric.WireupDelay] = { name: "Wireup Delay", unit: "ms"}; +map[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; +map[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px"}; +map[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; +map[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; +map[Metric.DocumentHeight] = { name: "Document Height", unit: "px"}; +map[Metric.ClickEvent] = { name: "Click Event", unit: ""}; +map[Metric.InteractionEvent] = { name: "Interaction Event", unit: ""}; + +export default function(tokens: Token[]): IDecodedMetric { + let metrics: IDecodedMetric = { counters: {}, measures: {}, events: [], marks: [], map }; let i = 0; let metricType = null; @@ -17,13 +35,10 @@ export default function(tokens: Token[]): IMetric { // Parse metrics switch (metricType) { case MetricType.Counter: - metrics.counter[tokens[i++] as Metric] = tokens[i++] as number; - break; - case MetricType.Timing: - metrics.timing[tokens[i++] as Metric] = { duration: tokens[i++] as number, count: tokens[i++] as number }; + metrics.counters[tokens[i++] as Metric] = tokens[i++] as number; break; case MetricType.Summary: - metrics.summary[tokens[i++] as Metric] = { + metrics.measures[tokens[i++] as Metric] = { sum: tokens[i++] as number, min: tokens[i++] as number, max: tokens[i++] as number, diff --git a/decode/render.ts b/decode/render.ts index b4ef9da9..17389d72 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,6 +1,5 @@ -import Metric from "../src/metrics/metric"; import { IDecodedNode } from "../types/dom"; -import { IMetric } from "../types/metrics"; +import { IDecodedMetric } from "../types/metrics"; import { IResizeViewport } from "../types/viewport"; let nodes = {}; @@ -10,56 +9,28 @@ export function reset(): void { nodes = {}; } -export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): void { - placeholder.removeAttribute("style"); - let margin = 10; - let px = "px"; - let width = data.width; - let height = data.height; - let availableWidth = (placeholder.contentWindow.innerWidth - (2 * margin)); - let scaleWidth = Math.min(availableWidth / width, 1); - let scaleHeight = Math.min((placeholder.contentWindow.innerHeight - (16 * margin)) / height, 1); - let scale = Math.min(scaleWidth, scaleHeight); - placeholder.style.position = "relative"; - placeholder.style.width = width + px; - placeholder.style.height = height + px; - placeholder.style.left = ((availableWidth - (width * scale)) / 2) + px; - placeholder.style.transformOrigin = "0 0 0"; - placeholder.style.transform = "scale(" + scale + ")"; - placeholder.style.border = "1px solid #cccccc"; - placeholder.style.margin = margin + px; - placeholder.style.overflow = "hidden"; -} - -export function metrics(data: IMetric, header: HTMLElement): void { +export function metrics(data: IDecodedMetric, header: HTMLElement): void { let html = []; // Counters - for (let metric in data.counter) { - if (data.counter[metric]) { - html.push(`

  • ${data.counter[metric]}

    ${getMetricName(parseInt(metric, 10))}
    • `); - } - } - - // Timing - for (let metric in data.timing) { - if (data.timing[metric]) { - html.push(`

  • ${data.timing[metric].duration}

    ${getMetricName(parseInt(metric, 10))}
    • `); + for (let metric in data.counters) { + if (data.counters[metric]) { + html.push(metricBox(data.counters[metric], data.map[metric].name, data.map[metric].unit)); } } // Summary - for (let metric in data.summary) { - if (data.summary[metric]) { - html.push(`

  • ${data.summary[metric].max}

    ${getMetricName(parseInt(metric, 10))}
    • `); + for (let metric in data.measures) { + if (data.measures[metric]) { + html.push(metricBox(data.measures[metric].max, data.map[metric].name, data.map[metric].unit)); } } header.innerHTML = `
      ${html.join("")}
    `; } -function getMetricName(metric: Metric): string { - return Metric[metric].replace(/([A-Z])/g, " $1").replace(/^./, function(str: string): string { return str.toUpperCase(); }); +function metricBox(value: number, name: string, unit: string): string { + return `

  • ${value}${unit}

    ${name}
    • `; } export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { @@ -175,3 +146,24 @@ function setAttributes(node: HTMLElement, attributes: object): void { } } } + +export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): void { + placeholder.removeAttribute("style"); + let margin = 10; + let px = "px"; + let width = data.width; + let height = data.height; + let availableWidth = (placeholder.contentWindow.innerWidth - (2 * margin)); + let scaleWidth = Math.min(availableWidth / width, 1); + let scaleHeight = Math.min((placeholder.contentWindow.innerHeight - (16 * margin)) / height, 1); + let scale = Math.min(scaleWidth, scaleHeight); + placeholder.style.position = "relative"; + placeholder.style.width = width + px; + placeholder.style.height = height + px; + placeholder.style.left = ((availableWidth - (width * scale)) / 2) + px; + placeholder.style.transformOrigin = "0 0 0"; + placeholder.style.transform = "scale(" + scale + ")"; + placeholder.style.border = "1px solid #cccccc"; + placeholder.style.margin = margin + px; + placeholder.style.overflow = "hidden"; +} diff --git a/src/core/task.ts b/src/core/task.ts index d6c704a4..37af0257 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,7 +1,7 @@ import {ITask } from "@clarity-types/core"; +import { Metric } from "@clarity-types/metrics"; import config from "@src/core/config"; import * as metrics from "@src/metrics"; -import Metric from "@src/metrics/metric"; let tracker: ITask = {}; let threshold = config.longtask; @@ -21,7 +21,7 @@ export function start(method: Metric): void { export function stop(method: Metric): void { let end = Date.now(); let duration = end - tracker[method]; - metrics.timing(method, duration); + metrics.measure(method, duration); } export async function idle(method: Metric): Promise { diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 88443edd..0ab0e321 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,10 +1,11 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; +import { Metric } from "@clarity-types/metrics"; import * as task from "@src/core/task"; import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "@src/dom/encode"; -import Metric from "@src/metrics/metric"; + import processNode from "./node"; export function start(): void { diff --git a/src/dom/encode.ts b/src/dom/encode.ts index ed1b2b54..53fc4344 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -1,10 +1,10 @@ import {Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; +import { Metric } from "@clarity-types/metrics"; import * as task from "@src/core/task"; import hash from "@src/data/hash"; import {check} from "@src/data/token"; import * as metrics from "@src/metrics"; -import Metric from "@src/metrics/metric"; import * as nodes from "./virtualdom"; @@ -25,7 +25,7 @@ export default async function(timer: Metric): Promise { if (data[key]) { switch (key) { case "tag": - metrics.counter(Metric.Nodes); + metrics.counter(Metric.NodeCount); markup.push(number(value.id)); if (value.parent) { markup.push(number(value.parent)); } if (value.next) { markup.push(number(value.next)); } diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index 9f523b70..c01d90d8 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,10 +1,10 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; +import { Metric } from "@clarity-types/metrics"; import * as task from "@src/core/task"; import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "@src/dom/encode"; -import Metric from "@src/metrics/metric"; import processNode from "./node"; let observer: MutationObserver; diff --git a/src/metrics/encode.ts b/src/metrics/encode.ts index 89cc63f5..bb13dd4c 100644 --- a/src/metrics/encode.ts +++ b/src/metrics/encode.ts @@ -7,7 +7,7 @@ export default function(): Token[] { // Encode counter metrics output.push(MetricType.Counter); - let counters = metrics.counter; + let counters = metrics.counters; for (let metric in counters) { if (counters[metric]) { output.push(parseInt(metric, 10)); @@ -17,7 +17,7 @@ export default function(): Token[] { // Encode summary metrics output.push(MetricType.Summary); - let summaries = metrics.summary; + let summaries = metrics.measures; for (let metric in summaries) { if (summaries[metric]) { let h = summaries[metric]; @@ -30,18 +30,6 @@ export default function(): Token[] { } } - // Encode timing metrics - output.push(MetricType.Timing); - let timings = metrics.timing; - for (let metric in timings) { - if (timings[metric]) { - let l = timings[metric]; - output.push(parseInt(metric, 10)); - output.push(l.duration); - output.push(l.count); - } - } - // Encode semantic events if (metrics.events.length > 0) { output.push(MetricType.Events); } let events = metrics.events; diff --git a/src/metrics/index.ts b/src/metrics/index.ts index 2eaafec9..41b40d68 100644 --- a/src/metrics/index.ts +++ b/src/metrics/index.ts @@ -1,11 +1,10 @@ -import { IMetric } from "@clarity-types/metrics"; +import { IMetric, Metric } from "@clarity-types/metrics"; import time from "@src/core/time"; -import Metric from "@src/metrics/metric"; export let metrics: IMetric = null; export function start(): void { - metrics = { counter: {}, timing: {}, summary: {}, events: [], marks: [] }; + metrics = { counters: {}, measures: {}, events: [], marks: [] }; } export function end(): void { @@ -13,23 +12,17 @@ export function end(): void { } export function counter(metric: Metric, increment: number = 1): void { - if (!(metric in metrics.counter)) { metrics.counter[metric] = 0; } - metrics.counter[metric] += increment; + if (!(metric in metrics.counters)) { metrics.counters[metric] = 0; } + metrics.counters[metric] += increment; } -export function timing(metric: Metric, duration: number): void { - if (!(metric in metrics.timing)) { metrics.timing[metric] = { duration: 0, count: 0 }; } - metrics.timing[metric].duration += duration; - metrics.timing[metric].count++; -} - -export function summary(metric: Metric, value: number): void { - if (!(metric in metrics.summary)) { metrics.summary[metric] = { sum: 0, min: null, max: null, sumsquared: 0, count: 0 }; } - metrics.summary[metric].sum += value; - metrics.summary[metric].min = metrics.summary[metric].min !== null ? Math.min(metrics.summary[metric].min, value) : value; - metrics.summary[metric].max = metrics.summary[metric].max !== null ? Math.max(metrics.summary[metric].max, value) : value; - metrics.summary[metric].sumsquared += value; - metrics.summary[metric].count++; +export function measure(metric: Metric, value: number): void { + if (!(metric in metrics.measures)) { metrics.measures[metric] = { sum: 0, min: null, max: null, sumsquared: 0, count: 0 }; } + metrics.measures[metric].sum += value; + metrics.measures[metric].min = metrics.measures[metric].min !== null ? Math.min(metrics.measures[metric].min, value) : value; + metrics.measures[metric].max = metrics.measures[metric].max !== null ? Math.max(metrics.measures[metric].max, value) : value; + metrics.measures[metric].sumsquared += (value * value); + metrics.measures[metric].count++; } export function event(metric: Metric, begin: number, duration: number = 0): void { diff --git a/src/metrics/metric.ts b/src/metrics/metric.ts deleted file mode 100644 index 738a1045..00000000 --- a/src/metrics/metric.ts +++ /dev/null @@ -1,24 +0,0 @@ -enum Metric { - /* Timing */ - DiscoverTime, - MutationTime, - WireupDelay, - ActiveTime, - /* Counter */ - Nodes, - Bytes, - Mutations, - Interactions, - Clicks, - Errors, - /* Summary */ - ViewportWidth, - ViewportHeight, - DocumentWidth, - DocumentHeight, - /* Semantic Events */ - Click, - Interaction -} - -export default Metric; diff --git a/types/data.d.ts b/types/data.d.ts index cc8689c1..d27f5a00 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,4 +1,4 @@ -import { IMetric } from "./metrics"; +import { IDecodedMetric } from "./metrics"; export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); @@ -39,7 +39,7 @@ export interface IPayload { export interface IDecodedPayload { envelope: IEnvelope; - metrics: IMetric; + metrics: IDecodedMetric; data: IDecodedEvent[]; } diff --git a/types/metrics.d.ts b/types/metrics.d.ts index f236b487..c329ffce 100644 --- a/types/metrics.d.ts +++ b/types/metrics.d.ts @@ -6,26 +6,51 @@ export const enum MetricType { Marks = "M" } +export const enum Metric { + /* Counter */ + NodeCount, + ByteCount, + MutationCount, + InteractionCount, + ClickCount, + ErrorCount, + /* Summary */ + DiscoverTime, + MutationTime, + WireupDelay, + ActiveTime, + ViewportWidth, + ViewportHeight, + DocumentWidth, + DocumentHeight, + /* Semantic Events */ + ClickEvent, + InteractionEvent +} + +export interface IMetricMap { + [key: number]: { + name: string; + unit: string; + }; +} + export interface IMetric { - timing: ITiming; - counter: ICounter; - summary: ISummary; + counters: ICounter; + measures: IMeasure; events: ISemanticEvent[]; marks: IMark[]; } -export interface ITiming { - [key: number]: { - duration: number; - count: number; - }; +export interface IDecodedMetric extends IMetric { + map: IMetricMap; } export interface ICounter { [key: number]: number; } -export interface ISummary { +export interface IMeasure { [key: number]: { sum: number; min: number; From 5769da00cf622ee4c62214354ad7bf09c5f78ab7 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 13 Aug 2019 09:45:24 -0700 Subject: [PATCH 037/105] Adding more metrics --- decode/metrics.ts | 12 ++++++------ decode/render.ts | 21 ++++++++++++++++----- src/clarity.ts | 4 ++-- src/data/metadata.ts | 8 ++++++-- src/data/upload.ts | 6 +++++- src/viewport/encode.ts | 6 ++++++ types/metrics.d.ts | 13 ++++++++----- 7 files changed, 49 insertions(+), 21 deletions(-) diff --git a/decode/metrics.ts b/decode/metrics.ts index 2aae4d7b..3c6e2c1f 100644 --- a/decode/metrics.ts +++ b/decode/metrics.ts @@ -4,19 +4,19 @@ import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metrics let map: IMetricMap = {}; map[Metric.NodeCount] = { name: "Node Count", unit: ""}; -map[Metric.ByteCount] = { name: "Byte Count", unit: ""}; +map[Metric.ByteCount] = { name: "Byte Count", unit: "KB"}; map[Metric.MutationCount] = { name: "Mutation Count", unit: ""}; map[Metric.InteractionCount] = { name: "Interaction Count", unit: ""}; map[Metric.ClickCount] = { name: "Click Count", unit: ""}; map[Metric.ErrorCount] = { name: "Error Count", unit: ""}; map[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; map[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; -map[Metric.WireupDelay] = { name: "Wireup Delay", unit: "ms"}; +map[Metric.WireupLag] = { name: "Wireup Delay", unit: "ms"}; map[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; -map[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px"}; -map[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; -map[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; -map[Metric.DocumentHeight] = { name: "Document Height", unit: "px"}; +map[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px", value: "max"}; +map[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px", value: "max"}; +map[Metric.DocumentWidth] = { name: "Document Width", unit: "px", value: "max"}; +map[Metric.DocumentHeight] = { name: "Document Height", unit: "px", value: "max"}; map[Metric.ClickEvent] = { name: "Click Event", unit: ""}; map[Metric.InteractionEvent] = { name: "Interaction Event", unit: ""}; diff --git a/decode/render.ts b/decode/render.ts index 17389d72..ffae4666 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ import { IDecodedNode } from "../types/dom"; -import { IDecodedMetric } from "../types/metrics"; +import { IDecodedMetric, IMetricMapValue } from "../types/metrics"; import { IResizeViewport } from "../types/viewport"; let nodes = {}; @@ -15,22 +15,33 @@ export function metrics(data: IDecodedMetric, header: HTMLElement): void { // Counters for (let metric in data.counters) { if (data.counters[metric]) { - html.push(metricBox(data.counters[metric], data.map[metric].name, data.map[metric].unit)); + let map = data.map[metric]; + let value = getValue(data.counters[metric], map.unit); + html.push(metricBox(value, data.map[metric])); } } // Summary for (let metric in data.measures) { if (data.measures[metric]) { - html.push(metricBox(data.measures[metric].max, data.map[metric].name, data.map[metric].unit)); + let m = data.measures[metric]; + let map = data.map[metric]; + let value = getValue(map.value ? m[map.value] : m.sum, map.unit); + html.push(metricBox(value, data.map[metric])); } } header.innerHTML = `
      ${html.join("")}
    `; } -function metricBox(value: number, name: string, unit: string): string { - return `

  • ${value}${unit}

    ${name}
    • `; +function getValue(value: number, unit: string): number { + switch (unit) { + case "KB": return Math.round(value / 1024); + } +} + +function metricBox(value: number, map: IMetricMapValue, metadata: string = null): string { + return `

  • ${value}${map.unit}
    ${metadata}

    ${map.name}
    • `; } export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { diff --git a/src/clarity.ts b/src/clarity.ts index d540c545..457cb16a 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -22,8 +22,8 @@ export function start(configuration: IConfig = {}): void { } event.reset(); - metadata.start(); metrics.start(); + metadata.start(); // DOM mutation.start(); @@ -45,8 +45,8 @@ export function start(configuration: IConfig = {}): void { export function end(): void { event.reset(); metadata.end(); - metrics.end(); mutation.end(); + metrics.end(); status = false; } diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 08992928..c489bf73 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,13 +1,17 @@ import { Event, Flush, IMetadata, Token } from "@clarity-types/data"; +import { Metric } from "@clarity-types/metrics"; import time from "@src/core/time"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; import queue from "@src/data/queue"; +import * as metrics from "@src/metrics"; export let metadata: IMetadata = null; export function start(): void { + metrics.measure(Metric.WireupLag, time()); + metadata = { sequence: 0, version, @@ -26,8 +30,8 @@ export function end(): void { } export function envelope(): Token[] { - metadata.sequence++; - return encode(true); + metadata.sequence++; + return encode(true); } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript diff --git a/src/data/upload.ts b/src/data/upload.ts index a20e8289..0f3e43f3 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,7 +1,9 @@ import { IEvent, IPayload } from "@clarity-types/data"; +import { Metric } from "@clarity-types/metrics"; import * as decode from "@decode/clarity"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; +import { measure } from "@src/metrics"; import metrics from "@src/metrics/encode"; export default function(events: IEvent[]): void { @@ -11,7 +13,9 @@ export default function(events: IEvent[]): void { d: events }; let upload = config.upload ? config.upload : send; - upload(JSON.stringify(payload)); + let data = JSON.stringify(payload); + measure(Metric.ByteCount, data.length); + upload(data); } function send(data: string): void { diff --git a/src/viewport/encode.ts b/src/viewport/encode.ts index 0b0dc9c7..f38bf9d1 100644 --- a/src/viewport/encode.ts +++ b/src/viewport/encode.ts @@ -1,4 +1,6 @@ import {Event, Token} from "@clarity-types/data"; +import {Metric} from "@clarity-types/metrics"; +import * as metrics from "@src/metrics"; import * as document from "./document"; import * as resize from "./resize"; import * as scroll from "./scroll"; @@ -12,12 +14,16 @@ export default function(type: Event): Token[] { let r = resize.data; tokens.push(r.width); tokens.push(r.height); + metrics.measure(Metric.ViewportWidth, r.width); + metrics.measure(Metric.ViewportHeight, r.height); resize.reset(); break; case Event.Document: let d = document.data; tokens.push(d.width); tokens.push(d.height); + metrics.measure(Metric.DocumentWidth, d.width); + metrics.measure(Metric.DocumentHeight, d.height); document.reset(); break; case Event.Scroll: diff --git a/types/metrics.d.ts b/types/metrics.d.ts index c329ffce..2b316f6f 100644 --- a/types/metrics.d.ts +++ b/types/metrics.d.ts @@ -17,7 +17,7 @@ export const enum Metric { /* Summary */ DiscoverTime, MutationTime, - WireupDelay, + WireupLag, ActiveTime, ViewportWidth, ViewportHeight, @@ -29,10 +29,13 @@ export const enum Metric { } export interface IMetricMap { - [key: number]: { - name: string; - unit: string; - }; + [key: number]: IMetricMapValue; +} + +export interface IMetricMapValue { + name: string; + unit: string; + value?: string; } export interface IMetric { From 46362279462d137dd1268115e2af37fcd3a310d3 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 13 Aug 2019 18:51:38 -0700 Subject: [PATCH 038/105] Metric presentation changes --- decode/dom.ts | 2 +- decode/render.ts | 34 ++++++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/decode/dom.ts b/decode/dom.ts index e54c4c33..6e40a7b0 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -77,7 +77,7 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { layout.push(parseInt(part, 36)); } layouts.push(layout); - } else if (output.tag === "*T" && parts.length === 2) { + } else if (output.tag === "*T" && parts.length === 2 && parts[0].length > 0) { let textCount = parseInt(parts[0], 36); let wordCount = parseInt(parts[1], 36); value = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); diff --git a/decode/render.ts b/decode/render.ts index ffae4666..ae6aed41 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -16,8 +16,8 @@ export function metrics(data: IDecodedMetric, header: HTMLElement): void { for (let metric in data.counters) { if (data.counters[metric]) { let map = data.map[metric]; - let value = getValue(data.counters[metric], map.unit); - html.push(metricBox(value, data.map[metric])); + let v = value(data.counters[metric], map.unit); + html.push(metricBox(v, data.map[metric])); } } @@ -26,22 +26,26 @@ export function metrics(data: IDecodedMetric, header: HTMLElement): void { if (data.measures[metric]) { let m = data.measures[metric]; let map = data.map[metric]; - let value = getValue(map.value ? m[map.value] : m.sum, map.unit); - html.push(metricBox(value, data.map[metric])); + let unit = map.unit; + let v = value(map.value ? m[map.value] : m.sum, unit); + let metadata = map.value === "max" ? `#${m.count} Min: ${value(m.min, unit)}` : `#${m.count} Max: ${value(m.max, unit)}`; + html.push(metricBox(v, data.map[metric], metadata)); } } header.innerHTML = `
      ${html.join("")}
    `; } -function getValue(value: number, unit: string): number { +function value(input: number, unit: string): number { switch (unit) { - case "KB": return Math.round(value / 1024); + case "KB": return Math.round(input / 1024); + default: return input; } } -function metricBox(value: number, map: IMetricMapValue, metadata: string = null): string { - return `

  • ${value}${map.unit}
    ${metadata}

    ${map.name}
    • `; +function metricBox(metric: number, map: IMetricMapValue, metadata: string = null): string { + metadata = metadata || ""; + return `

  • ${metric}${map.unit}
    ${metadata}

    ${map.name}
    • `; } export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { @@ -133,7 +137,12 @@ function element(nodeId: number): Node { function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void { if (parent !== null) { next = next && next.parentElement !== parent ? null : next; - parent.insertBefore(node, next); + try { + parent.insertBefore(node, next); + } catch (ex) { + console.error("Node: " + node + " | Parent: " + parent); + console.error("Exception encountered while inserting node: " + ex); + } } else if (parent === null && node.parentElement !== null) { node.parentElement.removeChild(node); } @@ -153,7 +162,12 @@ function setAttributes(node: HTMLElement, attributes: object): void { // Add new attributes for (let attribute in attributes) { if (attributes[attribute] !== undefined) { - node.setAttribute(attribute, attributes[attribute]); + try { + node.setAttribute(attribute, attributes[attribute]); + } catch (ex) { + console.error("Node: " + node + " | " + JSON.stringify(attributes)); + console.error("Exception encountered while adding attributes: " + ex); + } } } } From d87003c14f88760f98e8f06a0c6f09bbca35cc2c Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 13 Aug 2019 20:31:46 -0700 Subject: [PATCH 039/105] Adding scroll support to visualization --- decode/clarity.ts | 3 +++ decode/render.ts | 6 +++++- src/data/queue.ts | 5 ++--- src/interactions/mouse.ts | 6 ++---- src/viewport/scroll.ts | 7 +++---- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 164b1658..c00264e2 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -73,6 +73,9 @@ export function render(data: string, iframe: HTMLIFrameElement, header?: HTMLEle case Event.Resize: r.resize(entry.data[0], iframe); break; + case Event.Scroll: + r.scroll(entry.data[0], iframe); + break; } } } diff --git a/decode/render.ts b/decode/render.ts index ae6aed41..0d6f5b5b 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,6 +1,6 @@ import { IDecodedNode } from "../types/dom"; import { IDecodedMetric, IMetricMapValue } from "../types/metrics"; -import { IResizeViewport } from "../types/viewport"; +import { IResizeViewport, IScrollViewport } from "../types/viewport"; let nodes = {}; let svgns: string = "http://www.w3.org/2000/svg"; @@ -172,6 +172,10 @@ function setAttributes(node: HTMLElement, attributes: object): void { } } +export function scroll(data: IScrollViewport, placeholder: HTMLIFrameElement): void { + placeholder.contentWindow.scrollTo(data.x, data.y); +} + export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): void { placeholder.removeAttribute("style"); let margin = 10; diff --git a/src/data/queue.ts b/src/data/queue.ts index 17578320..6a0941cc 100644 --- a/src/data/queue.ts +++ b/src/data/queue.ts @@ -1,20 +1,19 @@ import { Event, Flush, IEvent, Token } from "@clarity-types/data"; +import config from "@src/core/config"; import upload from "@src/data/upload"; import recompute from "../core/recompute"; let events: IEvent[] = []; -let wait = 1000; let timeout: number = null; window["PAYLOAD"] = []; export default function(timestamp: number, event: Event, data: Token[], flush: Flush = Flush.Schedule): void { events.push({t: timestamp, e: event, d: data}); - switch (flush) { case Flush.Schedule: clearTimeout(timeout); - timeout = window.setTimeout(dequeue, wait); + timeout = window.setTimeout(dequeue, config.delay); break; case Flush.Force: clearTimeout(timeout); diff --git a/src/interactions/mouse.ts b/src/interactions/mouse.ts index 25e24c25..04f11fe6 100644 --- a/src/interactions/mouse.ts +++ b/src/interactions/mouse.ts @@ -8,8 +8,6 @@ import { getId } from "@src/dom/virtualdom"; import encode from "./encode"; let data: IMouseInteraction[] = []; -let wait = config.lookahead; -let distance = config.distance; let timeout: number = null; let timestamp: number = null; @@ -32,7 +30,7 @@ function handler(type: Mouse, evt: MouseEvent): void { buttons: evt.buttons }); if (timeout) { clearTimeout(timeout); } - timeout = window.setTimeout(schedule, wait); + timeout = window.setTimeout(schedule, config.lookahead); } function schedule(): void { @@ -64,5 +62,5 @@ export function summarize(): IMouseInteraction[] { function checkDistance(last: IMouseInteraction, current: IMouseInteraction): boolean { let dx = last.x - current.x; let dy = last.y - current.y; - return (dx * dx + dy * dy > distance * distance); + return (dx * dx + dy * dy > config.distance * config.distance); } diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts index 72100d9e..49c2b777 100644 --- a/src/viewport/scroll.ts +++ b/src/viewport/scroll.ts @@ -1,13 +1,12 @@ import { Event } from "@clarity-types/data"; import { IScrollViewport } from "@clarity-types/viewport"; +import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "./encode"; let data: IScrollViewport[] = []; -let wait = 1000; -let distance = 20; let timeout: number = null; let timestamp: number = null; @@ -21,7 +20,7 @@ function recompute(): void { let y = "pageYOffset" in window ? window.pageYOffset : document.documentElement.scrollTop; data.push({ time: time(), x, y }); if (timeout) { clearTimeout(timeout); } - timeout = window.setTimeout(schedule, wait); + timeout = window.setTimeout(schedule, config.lookahead); } function schedule(): void { @@ -53,5 +52,5 @@ export function summarize(): IScrollViewport[] { function checkDistance(last: IScrollViewport, current: IScrollViewport): boolean { let dx = last.x - current.x; let dy = last.y - current.y; - return (dx * dx + dy * dy > distance * distance); + return (dx * dx + dy * dy > config.distance * config.distance); } From e57325d02f59a54ee57230aaebe3a2e54a482831 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 14 Aug 2019 19:15:32 -0700 Subject: [PATCH 040/105] Rename and refactoring --- decode/metrics.ts | 2 +- decode/render.ts | 2 +- src/clarity.ts | 55 ------------------ src/core/task.ts | 4 +- src/data/encode.ts | 1 + src/data/metadata.ts | 4 +- src/data/upload.ts | 6 +- .../keyboard.ts => diagnostic/error.ts} | 0 .../selection.ts => diagnostic/image.ts} | 0 src/dom/discover.ts | 2 +- src/dom/encode.ts | 4 +- src/dom/mutation.ts | 2 +- src/index.ts | 56 ++++++++++++++++++- src/{interactions => interaction}/encode.ts | 0 .../touch.ts => interaction/keyboard.ts} | 0 src/{interactions => interaction}/mouse.ts | 2 +- src/interaction/selection.ts | 0 src/interaction/touch.ts | 0 src/{metrics => metric}/encode.ts | 4 +- src/{metrics => metric}/index.ts | 2 +- src/viewport/encode.ts | 4 +- types/data.d.ts | 2 +- types/index.d.ts | 4 +- types/{interactions.d.ts => interaction.d.ts} | 0 types/{metrics.d.ts => metric.d.ts} | 0 webpack/globalize.ts | 2 +- 26 files changed, 78 insertions(+), 80 deletions(-) delete mode 100644 src/clarity.ts rename src/{interactions/keyboard.ts => diagnostic/error.ts} (100%) rename src/{interactions/selection.ts => diagnostic/image.ts} (100%) rename src/{interactions => interaction}/encode.ts (100%) rename src/{interactions/touch.ts => interaction/keyboard.ts} (100%) rename src/{interactions => interaction}/mouse.ts (99%) create mode 100644 src/interaction/selection.ts create mode 100644 src/interaction/touch.ts rename src/{metrics => metric}/encode.ts (93%) rename src/{metrics => metric}/index.ts (95%) rename types/{interactions.d.ts => interaction.d.ts} (100%) rename types/{metrics.d.ts => metric.d.ts} (100%) diff --git a/decode/metrics.ts b/decode/metrics.ts index 3c6e2c1f..5557590b 100644 --- a/decode/metrics.ts +++ b/decode/metrics.ts @@ -1,5 +1,5 @@ import { Token } from "../types/data"; -import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metrics"; +import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metric"; let map: IMetricMap = {}; diff --git a/decode/render.ts b/decode/render.ts index 0d6f5b5b..5e21a40c 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ import { IDecodedNode } from "../types/dom"; -import { IDecodedMetric, IMetricMapValue } from "../types/metrics"; +import { IDecodedMetric, IMetricMapValue } from "../types/metric"; import { IResizeViewport, IScrollViewport } from "../types/viewport"; let nodes = {}; diff --git a/src/clarity.ts b/src/clarity.ts deleted file mode 100644 index 457cb16a..00000000 --- a/src/clarity.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { IConfig } from "@clarity-types/core"; -import config from "@src/core/config"; -import * as event from "@src/core/event"; -import * as metadata from "@src/data/metadata"; -import * as discover from "@src/dom/discover"; -import * as mutation from "@src/dom/mutation"; -import * as mouse from "@src/interactions/mouse"; -import * as metrics from "@src/metrics"; -import * as document from "@src/viewport/document"; -import * as resize from "@src/viewport/resize"; -import * as scroll from "@src/viewport/scroll"; -import * as visibility from "@src/viewport/visibility"; - -let status = false; - -/* Initial discovery of DOM */ -export function start(configuration: IConfig = {}): void { - - // Process custom configuration, if available - for (let key in configuration) { - if (key in config) { config[key] = configuration[key]; } - } - - event.reset(); - metrics.start(); - metadata.start(); - - // DOM - mutation.start(); - discover.start(); - - // Viewport - document.start(); - resize.start(); - visibility.start(); - scroll.start(); - - // Pointer - mouse.start(); - - // Mark Clarity session as active - status = true; -} - -export function end(): void { - event.reset(); - metadata.end(); - mutation.end(); - metrics.end(); - status = false; -} - -export function active(): boolean { - return status; -} diff --git a/src/core/task.ts b/src/core/task.ts index 37af0257..33ea4e81 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,7 +1,7 @@ import {ITask } from "@clarity-types/core"; -import { Metric } from "@clarity-types/metrics"; +import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; -import * as metrics from "@src/metrics"; +import * as metrics from "@src/metric"; let tracker: ITask = {}; let threshold = config.longtask; diff --git a/src/data/encode.ts b/src/data/encode.ts index d482a363..c0edfd08 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -3,6 +3,7 @@ import { metadata } from "@src/data/metadata"; export default function(envelope: boolean = false): Token[] { let tokens = []; + tokens.push(metadata.sequence); tokens.push(metadata.version); tokens.push(metadata.pageId); tokens.push(metadata.userId); diff --git a/src/data/metadata.ts b/src/data/metadata.ts index c489bf73..2f5138d2 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,11 +1,11 @@ import { Event, Flush, IMetadata, Token } from "@clarity-types/data"; -import { Metric } from "@clarity-types/metrics"; +import { Metric } from "@clarity-types/metric"; import time from "@src/core/time"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; import queue from "@src/data/queue"; -import * as metrics from "@src/metrics"; +import * as metrics from "@src/metric"; export let metadata: IMetadata = null; diff --git a/src/data/upload.ts b/src/data/upload.ts index 0f3e43f3..44338c8d 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,10 +1,10 @@ import { IEvent, IPayload } from "@clarity-types/data"; -import { Metric } from "@clarity-types/metrics"; +import { Metric } from "@clarity-types/metric"; import * as decode from "@decode/clarity"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; -import { measure } from "@src/metrics"; -import metrics from "@src/metrics/encode"; +import { measure } from "@src/metric"; +import metrics from "@src/metric/encode"; export default function(events: IEvent[]): void { let payload: IPayload = { diff --git a/src/interactions/keyboard.ts b/src/diagnostic/error.ts similarity index 100% rename from src/interactions/keyboard.ts rename to src/diagnostic/error.ts diff --git a/src/interactions/selection.ts b/src/diagnostic/image.ts similarity index 100% rename from src/interactions/selection.ts rename to src/diagnostic/image.ts diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 0ab0e321..f61b0643 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -1,6 +1,6 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; -import { Metric } from "@clarity-types/metrics"; +import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import time from "@src/core/time"; import queue from "@src/data/queue"; diff --git a/src/dom/encode.ts b/src/dom/encode.ts index 53fc4344..ccf8511a 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -1,10 +1,10 @@ import {Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; -import { Metric } from "@clarity-types/metrics"; +import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import hash from "@src/data/hash"; import {check} from "@src/data/token"; -import * as metrics from "@src/metrics"; +import * as metrics from "@src/metric"; import * as nodes from "./virtualdom"; diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index c01d90d8..be4e9e26 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -1,6 +1,6 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; -import { Metric } from "@clarity-types/metrics"; +import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import time from "@src/core/time"; import queue from "@src/data/queue"; diff --git a/src/index.ts b/src/index.ts index e81ae3fe..eb6d9687 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,55 @@ -import * as clarity from "./clarity"; +import { IConfig } from "@clarity-types/core"; +import config from "@src/core/config"; +import * as event from "@src/core/event"; +import * as metadata from "@src/data/metadata"; +import * as discover from "@src/dom/discover"; +import * as mutation from "@src/dom/mutation"; +import * as mouse from "@src/interaction/mouse"; +import * as metrics from "@src/metric"; +import * as document from "@src/viewport/document"; +import * as resize from "@src/viewport/resize"; +import * as scroll from "@src/viewport/scroll"; +import * as visibility from "@src/viewport/visibility"; -export { clarity }; +let status = false; + +/* Initial discovery of DOM */ +export function start(configuration: IConfig = {}): void { + + // Process custom configuration, if available + for (let key in configuration) { + if (key in config) { config[key] = configuration[key]; } + } + + event.reset(); + metrics.start(); + metadata.start(); + + // DOM + mutation.start(); + discover.start(); + + // Viewport + document.start(); + resize.start(); + visibility.start(); + scroll.start(); + + // Pointer + mouse.start(); + + // Mark Clarity session as active + status = true; +} + +export function end(): void { + event.reset(); + metadata.end(); + mutation.end(); + metrics.end(); + status = false; +} + +export function active(): boolean { + return status; +} diff --git a/src/interactions/encode.ts b/src/interaction/encode.ts similarity index 100% rename from src/interactions/encode.ts rename to src/interaction/encode.ts diff --git a/src/interactions/touch.ts b/src/interaction/keyboard.ts similarity index 100% rename from src/interactions/touch.ts rename to src/interaction/keyboard.ts diff --git a/src/interactions/mouse.ts b/src/interaction/mouse.ts similarity index 99% rename from src/interactions/mouse.ts rename to src/interaction/mouse.ts index 04f11fe6..5a99da7a 100644 --- a/src/interactions/mouse.ts +++ b/src/interaction/mouse.ts @@ -1,5 +1,5 @@ import { Event } from "@clarity-types/data"; -import { IMouseInteraction, Mouse } from "@clarity-types/interactions"; +import { IMouseInteraction, Mouse } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; diff --git a/src/interaction/selection.ts b/src/interaction/selection.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/interaction/touch.ts b/src/interaction/touch.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/metrics/encode.ts b/src/metric/encode.ts similarity index 93% rename from src/metrics/encode.ts rename to src/metric/encode.ts index bb13dd4c..2c1b9645 100644 --- a/src/metrics/encode.ts +++ b/src/metric/encode.ts @@ -1,6 +1,6 @@ import {Token} from "@clarity-types/data"; -import { MetricType } from "@clarity-types/metrics"; -import { metrics } from "@src/metrics"; +import { MetricType } from "@clarity-types/metric"; +import { metrics } from "@src/metric"; export default function(): Token[] { let output = []; diff --git a/src/metrics/index.ts b/src/metric/index.ts similarity index 95% rename from src/metrics/index.ts rename to src/metric/index.ts index 41b40d68..077c40cd 100644 --- a/src/metrics/index.ts +++ b/src/metric/index.ts @@ -1,4 +1,4 @@ -import { IMetric, Metric } from "@clarity-types/metrics"; +import { IMetric, Metric } from "@clarity-types/metric"; import time from "@src/core/time"; export let metrics: IMetric = null; diff --git a/src/viewport/encode.ts b/src/viewport/encode.ts index f38bf9d1..7d2b56d7 100644 --- a/src/viewport/encode.ts +++ b/src/viewport/encode.ts @@ -1,6 +1,6 @@ import {Event, Token} from "@clarity-types/data"; -import {Metric} from "@clarity-types/metrics"; -import * as metrics from "@src/metrics"; +import {Metric} from "@clarity-types/metric"; +import * as metrics from "@src/metric"; import * as document from "./document"; import * as resize from "./resize"; import * as scroll from "./scroll"; diff --git a/types/data.d.ts b/types/data.d.ts index d27f5a00..b7d7e921 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,4 +1,4 @@ -import { IDecodedMetric } from "./metrics"; +import { IDecodedMetric } from "./metric"; export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); diff --git a/types/index.d.ts b/types/index.d.ts index 0e4947b6..4742ca7f 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -11,8 +11,8 @@ declare const clarity: IClarityJs; export * from "./data"; export * from "./dom"; -export * from "./interactions"; -export * from "./metrics"; +export * from "./interaction"; +export * from "./metric"; export * from "./viewport"; export { clarity }; diff --git a/types/interactions.d.ts b/types/interaction.d.ts similarity index 100% rename from types/interactions.d.ts rename to types/interaction.d.ts diff --git a/types/metrics.d.ts b/types/metric.d.ts similarity index 100% rename from types/metrics.d.ts rename to types/metric.d.ts diff --git a/webpack/globalize.ts b/webpack/globalize.ts index 21c0e025..b2493af2 100644 --- a/webpack/globalize.ts +++ b/webpack/globalize.ts @@ -1,4 +1,4 @@ -import * as clarity from "@src/clarity"; +import * as clarity from "@src/index"; // When built with webpack for prod, compiled clarity-js bundle doesn't expose the module anywhere on the page. // Since we need clarity-js to be available globally, we can create a wrapper module that would assign clarity to window. From 92336ca927e9589a76987f8cdffc43813af183b7 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 15 Aug 2019 08:06:31 -0700 Subject: [PATCH 041/105] Adding diagnostic support --- decode/clarity.ts | 4 +-- decode/{metrics.ts => metric.ts} | 3 +- src/clarity.ts | 58 ++++++++++++++++++++++++++++++++ src/core/event.ts | 4 +-- src/data/metadata.ts | 4 +-- src/diagnostic/encode.ts | 35 +++++++++++++++++++ src/diagnostic/error.ts | 0 src/diagnostic/image.ts | 33 ++++++++++++++++++ src/diagnostic/index.ts | 12 +++++++ src/diagnostic/script.ts | 34 +++++++++++++++++++ src/dom/node.ts | 18 +++++----- src/dom/virtualdom.ts | 14 ++++---- src/index.ts | 56 ++---------------------------- src/interaction/mouse.ts | 2 +- src/viewport/encode.ts | 10 +++--- types/data.d.ts | 4 ++- types/diagnostic.d.ts | 12 +++++++ types/metric.d.ts | 3 +- webpack/globalize.ts | 2 +- 19 files changed, 222 insertions(+), 86 deletions(-) rename decode/{metrics.ts => metric.ts} (95%) create mode 100644 src/clarity.ts create mode 100644 src/diagnostic/encode.ts delete mode 100644 src/diagnostic/error.ts create mode 100644 src/diagnostic/index.ts create mode 100644 src/diagnostic/script.ts create mode 100644 types/diagnostic.d.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index c00264e2..e3a90305 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -2,7 +2,7 @@ import { Event, IDecodedEvent, IDecodedPayload, IEvent, IPayload } from "../type import dom from "./dom"; import envelope from "./envelope"; import metadata from "./metadata"; -import metrics from "./metrics"; +import metric from "./metric"; import * as r from "./render"; import viewport from "./viewport"; @@ -13,7 +13,7 @@ export function json(data: string): IDecodedPayload { let payload = JSON.parse(data); let decoded: IDecodedPayload = { envelope: envelope(payload.e), - metrics: metrics(payload.m), + metrics: metric(payload.m), data: null }; diff --git a/decode/metrics.ts b/decode/metric.ts similarity index 95% rename from decode/metrics.ts rename to decode/metric.ts index 5557590b..bec8e95f 100644 --- a/decode/metrics.ts +++ b/decode/metric.ts @@ -8,7 +8,8 @@ map[Metric.ByteCount] = { name: "Byte Count", unit: "KB"}; map[Metric.MutationCount] = { name: "Mutation Count", unit: ""}; map[Metric.InteractionCount] = { name: "Interaction Count", unit: ""}; map[Metric.ClickCount] = { name: "Click Count", unit: ""}; -map[Metric.ErrorCount] = { name: "Error Count", unit: ""}; +map[Metric.ScriptErrorCount] = { name: "Script Errors", unit: ""}; +map[Metric.ImageErrorCount] = { name: "Image Errors", unit: ""}; map[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; map[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; map[Metric.WireupLag] = { name: "Wireup Delay", unit: "ms"}; diff --git a/src/clarity.ts b/src/clarity.ts new file mode 100644 index 00000000..6757ad30 --- /dev/null +++ b/src/clarity.ts @@ -0,0 +1,58 @@ +import { IConfig } from "@clarity-types/core"; +import config from "@src/core/config"; +import * as event from "@src/core/event"; +import * as metadata from "@src/data/metadata"; +import * as diagnostic from "@src/diagnostic"; +import * as discover from "@src/dom/discover"; +import * as mutation from "@src/dom/mutation"; +import * as mouse from "@src/interaction/mouse"; +import * as metric from "@src/metric"; +import * as document from "@src/viewport/document"; +import * as resize from "@src/viewport/resize"; +import * as scroll from "@src/viewport/scroll"; +import * as visibility from "@src/viewport/visibility"; + +let status = false; + +/* Initial discovery of DOM */ +export function start(configuration: IConfig = {}): void { + + // Process custom configuration, if available + for (let key in configuration) { + if (key in config) { config[key] = configuration[key]; } + } + + event.reset(); + metric.start(); + metadata.start(); + diagnostic.start(); + + // DOM + mutation.start(); + discover.start(); + + // Viewport + document.start(); + resize.start(); + visibility.start(); + scroll.start(); + + // Pointer + mouse.start(); + + // Mark Clarity session as active + status = true; +} + +export function end(): void { + event.reset(); + metadata.end(); + mutation.end(); + metric.end(); + diagnostic.end(); + status = false; +} + +export function active(): boolean { + return status; +} diff --git a/src/core/event.ts b/src/core/event.ts index c6638975..9a37c1a7 100644 --- a/src/core/event.ts +++ b/src/core/event.ts @@ -2,9 +2,9 @@ import { IBindingContainer, IEventBindingPair } from "@clarity-types/core"; let bindings: IBindingContainer = {}; -export function bind(target: EventTarget, event: string, listener: EventListener): void { +export function bind(target: EventTarget, event: string, listener: EventListener, useCapture: boolean = false): void { let eventBindings = bindings[event] || []; - target.addEventListener(event, listener, false); + target.addEventListener(event, listener, useCapture); eventBindings.push({ target, listener diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 2f5138d2..29ff25b3 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -5,12 +5,12 @@ import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; import queue from "@src/data/queue"; -import * as metrics from "@src/metric"; +import * as metric from "@src/metric"; export let metadata: IMetadata = null; export function start(): void { - metrics.measure(Metric.WireupLag, time()); + metric.measure(Metric.WireupLag, time()); metadata = { sequence: 0, diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts new file mode 100644 index 00000000..80b70f9f --- /dev/null +++ b/src/diagnostic/encode.ts @@ -0,0 +1,35 @@ +import {Event, Token} from "@clarity-types/data"; +import {Metric} from "@clarity-types/metric"; +import * as image from "@src/diagnostic/image"; +import * as script from "@src/diagnostic/script"; +import * as metric from "@src/metric"; + +export default function(type: Event): Token[] { + let tokens = []; + + switch (type) { + case Event.ScriptError: + let scripts = script.data; + for (let e of scripts) { + tokens.push(e.source); + tokens.push(e.message); + tokens.push(e.line); + tokens.push(e.column); + tokens.push(e.stack); + metric.counter(Metric.ScriptErrorCount); + } + script.reset(); + break; + case Event.ImageError: + let images = image.data; + for (let e of images) { + tokens.push(e.source); + tokens.push(e.target); + metric.counter(Metric.ImageErrorCount); + } + image.reset(); + break; + } + + return tokens; +} diff --git a/src/diagnostic/error.ts b/src/diagnostic/error.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/diagnostic/image.ts b/src/diagnostic/image.ts index e69de29b..ededd879 100644 --- a/src/diagnostic/image.ts +++ b/src/diagnostic/image.ts @@ -0,0 +1,33 @@ +import { Event } from "@clarity-types/data"; +import { IImageError } from "@clarity-types/diagnostic"; +import { bind } from "@src/core/event"; +import time from "@src/core/time"; +import queue from "@src/data/queue"; +import { getId } from "@src/dom/virtualdom"; +import encode from "./encode"; + +export let data: IImageError[] = []; + +export function start(): void { + bind(document, "error", handler, true); +} + +export function end(): void { + return; +} + +function handler(error: ErrorEvent): void { + let target = error.target as HTMLElement; + if (target && target.tagName === "IMG") { + data.push({ + source: error["filename"], + target: getId(target) + }); + } + + queue(time(), Event.ImageError, encode(Event.ImageError)); +} + +export function reset(): void { + data = []; +} diff --git a/src/diagnostic/index.ts b/src/diagnostic/index.ts new file mode 100644 index 00000000..d3cd1def --- /dev/null +++ b/src/diagnostic/index.ts @@ -0,0 +1,12 @@ +import * as image from "./image"; +import * as script from "./script"; + +export function start(): void { + script.start(); + image.start(); +} + +export function end(): void { + script.end(); + image.end(); +} diff --git a/src/diagnostic/script.ts b/src/diagnostic/script.ts new file mode 100644 index 00000000..c56e6945 --- /dev/null +++ b/src/diagnostic/script.ts @@ -0,0 +1,34 @@ +import { Event } from "@clarity-types/data"; +import { IScriptError } from "@clarity-types/diagnostic"; +import { bind } from "@src/core/event"; +import time from "@src/core/time"; +import queue from "@src/data/queue"; +import encode from "./encode"; + +export let data: IScriptError[] = []; + +export function start(): void { + bind(window, "error", handler); +} + +export function end(): void { + return; +} + +function handler(error: ErrorEvent): void { + let e = error["error"] || error; + + data.push({ + message: e.message, + stack: e.stack, + line: error["lineno"], + column: error["colno"], + source: error["filename"] + }); + + queue(time(), Event.ScriptError, encode(Event.ScriptError)); +} + +export function reset(): void { + data = []; +} diff --git a/src/dom/node.ts b/src/dom/node.ts index dedd6878..0def7ab4 100644 --- a/src/dom/node.ts +++ b/src/dom/node.ts @@ -1,20 +1,20 @@ import { Source } from "@clarity-types/dom"; import config from "@src/core/config"; -import * as nodes from "./virtualdom"; +import * as virtualdom from "./virtualdom"; let ignoreAttributes = ["title", "alt", "onload", "onfocus"]; export default function(node: Node, source: Source): void { // Do not track this change if we are attempting to remove a node before discovering it - if (source === Source.ChildListRemove && nodes.has(node) === false) { return; } + if (source === Source.ChildListRemove && virtualdom.has(node) === false) { return; } - let call = nodes.has(node) ? "update" : "add"; + let call = virtualdom.has(node) ? "update" : "add"; switch (node.nodeType) { case Node.DOCUMENT_TYPE_NODE: let doctype = node as DocumentType; let docAttributes = { name: doctype.name, publicId: doctype.publicId, systemId: doctype.systemId }; let docData = { tag: "*D", attributes: docAttributes }; - nodes[call](node, docData, source); + virtualdom[call](node, docData, source); break; case Node.TEXT_NODE: // Account for this text node only if we are tracking the parent node @@ -23,10 +23,10 @@ export default function(node: Node, source: Source): void { // The only exception is when we receive a mutation to remove the text node, in that case // parent will be null, but we can still process the node by checking it's an update call. let parent = node.parentElement; - if (call === "update" || (parent && nodes.has(parent) && parent.tagName !== "STYLE")) { + if (call === "update" || (parent && virtualdom.has(parent) && parent.tagName !== "STYLE")) { let textData = { tag: "*T", value: node.nodeValue }; textData["layout"] = getTextLayout(node); - nodes[call](node, textData, source); + virtualdom[call](node, textData, source); } break; case Node.ELEMENT_NODE: @@ -41,17 +41,17 @@ export default function(node: Node, source: Source): void { let head = { tag, attributes: getAttributes(element.attributes) }; // Capture base href as part of discovering DOM if (call === "add") { head.attributes["*B"] = location.protocol + "//" + location.hostname; } - nodes[call](node, head, source); + virtualdom[call](node, head, source); break; case "STYLE": let attributes = getAttributes(element.attributes); let styleData = { tag, attributes, value: getStyleValue(element as HTMLStyleElement) }; - nodes[call](node, styleData, source); + virtualdom[call](node, styleData, source); break; default: let data = { tag, attributes: getAttributes(element.attributes) }; data["layout"] = getLayout(element); - nodes[call](node, data, source); + virtualdom[call](node, data, source); break; } break; diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 8bcf5c81..4a708a31 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -16,7 +16,7 @@ let backupValues: INodeValue[]; // For debugging window["DOM"] = { getId, get, getNode, changes }; -export function getId(node: Node, autogen: boolean = true): number { +export function getId(node: Node, autogen: boolean = false): number { if (node === null) { return null; } let id = node[NODE_ID_PROP]; if (!id && autogen) { @@ -26,8 +26,8 @@ export function getId(node: Node, autogen: boolean = true): number { } export function add(node: Node, data: INodeData, source: Source): void { - let id = getId(node); - let parentId = node.parentElement ? getId(node.parentElement, false) : null; + let id = getId(node, true); + let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); if (parentId >= 0 && values[parentId]) { @@ -46,8 +46,8 @@ export function add(node: Node, data: INodeData, source: Source): void { } export function update(node: Node, data: INodeData, source: Source): void { - let id = getId(node, false); - let parentId = node.parentElement ? getId(node.parentElement, false) : null; + let id = getId(node); + let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); if (id in values) { @@ -96,7 +96,7 @@ export function update(node: Node, data: INodeData, source: Source): void { function getNextId(node: Node): number { let id = null; while (id === null && node.nextSibling) { - id = getId(node.nextSibling, false); + id = getId(node.nextSibling); node = node.nextSibling; } return id; @@ -115,7 +115,7 @@ export function get(node: Node): INodeValue { } export function has(node: Node): boolean { - return getId(node, false) in nodes; + return getId(node) in nodes; } export function getNodes(): Node[] { diff --git a/src/index.ts b/src/index.ts index eb6d9687..e81ae3fe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,55 +1,3 @@ -import { IConfig } from "@clarity-types/core"; -import config from "@src/core/config"; -import * as event from "@src/core/event"; -import * as metadata from "@src/data/metadata"; -import * as discover from "@src/dom/discover"; -import * as mutation from "@src/dom/mutation"; -import * as mouse from "@src/interaction/mouse"; -import * as metrics from "@src/metric"; -import * as document from "@src/viewport/document"; -import * as resize from "@src/viewport/resize"; -import * as scroll from "@src/viewport/scroll"; -import * as visibility from "@src/viewport/visibility"; +import * as clarity from "./clarity"; -let status = false; - -/* Initial discovery of DOM */ -export function start(configuration: IConfig = {}): void { - - // Process custom configuration, if available - for (let key in configuration) { - if (key in config) { config[key] = configuration[key]; } - } - - event.reset(); - metrics.start(); - metadata.start(); - - // DOM - mutation.start(); - discover.start(); - - // Viewport - document.start(); - resize.start(); - visibility.start(); - scroll.start(); - - // Pointer - mouse.start(); - - // Mark Clarity session as active - status = true; -} - -export function end(): void { - event.reset(); - metadata.end(); - mutation.end(); - metrics.end(); - status = false; -} - -export function active(): boolean { - return status; -} +export { clarity }; diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index 5a99da7a..f1da8fdc 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -26,7 +26,7 @@ function handler(type: Mouse, evt: MouseEvent): void { type, x: "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null), y: "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null), - target: evt.target ? getId(evt.target as Node, false) : null, + target: evt.target ? getId(evt.target as Node) : null, buttons: evt.buttons }); if (timeout) { clearTimeout(timeout); } diff --git a/src/viewport/encode.ts b/src/viewport/encode.ts index 7d2b56d7..68351906 100644 --- a/src/viewport/encode.ts +++ b/src/viewport/encode.ts @@ -1,6 +1,6 @@ import {Event, Token} from "@clarity-types/data"; import {Metric} from "@clarity-types/metric"; -import * as metrics from "@src/metric"; +import * as metric from "@src/metric"; import * as document from "./document"; import * as resize from "./resize"; import * as scroll from "./scroll"; @@ -14,16 +14,16 @@ export default function(type: Event): Token[] { let r = resize.data; tokens.push(r.width); tokens.push(r.height); - metrics.measure(Metric.ViewportWidth, r.width); - metrics.measure(Metric.ViewportHeight, r.height); + metric.measure(Metric.ViewportWidth, r.width); + metric.measure(Metric.ViewportHeight, r.height); resize.reset(); break; case Event.Document: let d = document.data; tokens.push(d.width); tokens.push(d.height); - metrics.measure(Metric.DocumentWidth, d.width); - metrics.measure(Metric.DocumentHeight, d.height); + metric.measure(Metric.DocumentWidth, d.width); + metric.measure(Metric.DocumentHeight, d.height); document.reset(); break; case Event.Scroll: diff --git a/types/data.d.ts b/types/data.d.ts index b7d7e921..b0251ff6 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -16,7 +16,9 @@ export const enum Event { Document, Visibility, Network, - Performance + Performance, + ScriptError, + ImageError } export const enum Flush { diff --git a/types/diagnostic.d.ts b/types/diagnostic.d.ts new file mode 100644 index 00000000..27aa9b4d --- /dev/null +++ b/types/diagnostic.d.ts @@ -0,0 +1,12 @@ +export interface IScriptError { + source: string; + message: string; + line: number; + column: number; + stack: string; +} + +export interface IImageError { + source: string; + target: number; +} diff --git a/types/metric.d.ts b/types/metric.d.ts index 2b316f6f..56a1ac60 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -13,7 +13,8 @@ export const enum Metric { MutationCount, InteractionCount, ClickCount, - ErrorCount, + ScriptErrorCount, + ImageErrorCount, /* Summary */ DiscoverTime, MutationTime, diff --git a/webpack/globalize.ts b/webpack/globalize.ts index b2493af2..21c0e025 100644 --- a/webpack/globalize.ts +++ b/webpack/globalize.ts @@ -1,4 +1,4 @@ -import * as clarity from "@src/index"; +import * as clarity from "@src/clarity"; // When built with webpack for prod, compiled clarity-js bundle doesn't expose the module anywhere on the page. // Since we need clarity-js to be available globally, we can create a wrapper module that would assign clarity to window. From 2e2d0e46ef7e5df1221b482de207692c4003baed Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 15 Aug 2019 13:10:37 -0700 Subject: [PATCH 042/105] Adding summary support --- src/summary/index.ts | 0 types/data.d.ts | 3 +++ types/summary.d.ts | 9 +++++++++ 3 files changed, 12 insertions(+) create mode 100644 src/summary/index.ts create mode 100644 types/summary.d.ts diff --git a/src/summary/index.ts b/src/summary/index.ts new file mode 100644 index 00000000..e69de29b diff --git a/types/data.d.ts b/types/data.d.ts index b0251ff6..022c43dd 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,4 +1,5 @@ import { IDecodedMetric } from "./metric"; +import { ISummary } from "./summary"; export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); @@ -36,12 +37,14 @@ export interface IEvent { export interface IPayload { e: Token[]; m: Token[]; + s: Token[]; d: IEvent[]; } export interface IDecodedPayload { envelope: IEnvelope; metrics: IDecodedMetric; + Summary: ISummary; data: IDecodedEvent[]; } diff --git a/types/summary.d.ts b/types/summary.d.ts new file mode 100644 index 00000000..e2b2ef2f --- /dev/null +++ b/types/summary.d.ts @@ -0,0 +1,9 @@ +export interface ISummary { + time: number; + duration: number; +} + +export interface IScrollSummary extends ISummary { + distanceX: number; + distanceY: number; +} \ No newline at end of file From 5bd64a6a53cacd05363848ce2ca9cde5e2359e70 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 15 Aug 2019 13:10:53 -0700 Subject: [PATCH 043/105] Adding summary support --- types/summary.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/summary.d.ts b/types/summary.d.ts index e2b2ef2f..584c5a91 100644 --- a/types/summary.d.ts +++ b/types/summary.d.ts @@ -6,4 +6,4 @@ export interface ISummary { export interface IScrollSummary extends ISummary { distanceX: number; distanceY: number; -} \ No newline at end of file +} From dcb6a3b1ddd4ed5907e51546d89e9dd059bfaa71 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 16 Aug 2019 07:29:26 -0700 Subject: [PATCH 044/105] Breaking events into two queues --- decode/clarity.ts | 6 +++++- src/data/queue.ts | 25 +++++++++++++++++++++---- src/data/upload.ts | 7 ++++--- src/summary/index.ts | 0 types/data.d.ts | 11 +++++++---- types/summary.d.ts | 9 --------- 6 files changed, 37 insertions(+), 21 deletions(-) delete mode 100644 src/summary/index.ts delete mode 100644 types/summary.d.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index e3a90305..6f3318d2 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -24,7 +24,7 @@ export function json(data: string): IDecodedPayload { } let events: IDecodedEvent[] = []; - let encoded: IEvent[] = payload.d; + let encoded: IEvent[] = merge(payload.s, payload.c); payloads.push(payload); for (let entry of encoded) { @@ -79,3 +79,7 @@ export function render(data: string, iframe: HTMLIFrameElement, header?: HTMLEle } } } + +function merge(one: IEvent[], two: IEvent[]): IEvent[] { + return one.concat(two).sort(function(a: IEvent, b: IEvent): number { return a.t - b.t; }); +} diff --git a/src/data/queue.ts b/src/data/queue.ts index 6a0941cc..be46aee3 100644 --- a/src/data/queue.ts +++ b/src/data/queue.ts @@ -1,15 +1,32 @@ -import { Event, Flush, IEvent, Token } from "@clarity-types/data"; +import { Event, Flush, IEventQueue, Token } from "@clarity-types/data"; import config from "@src/core/config"; import upload from "@src/data/upload"; import recompute from "../core/recompute"; -let events: IEvent[] = []; +let events: IEventQueue = { server: [], client: [] }; let timeout: number = null; window["PAYLOAD"] = []; export default function(timestamp: number, event: Event, data: Token[], flush: Flush = Flush.Schedule): void { - events.push({t: timestamp, e: event, d: data}); + let e = {t: timestamp, e: event, d: data}; + + switch (event) { + case Event.Mouse: + case Event.Touch: + case Event.Keyboard: + case Event.Selection: + case Event.Resize: + case Event.Scroll: + case Event.Document: + case Event.Visibility: + events.server.push(e); + break; + default: + events.client.push(e); + break; + } + switch (flush) { case Flush.Schedule: clearTimeout(timeout); @@ -29,5 +46,5 @@ function dequeue(): void { } function reset(): void { - events = []; + events = { server: [], client: [] }; } diff --git a/src/data/upload.ts b/src/data/upload.ts index 44338c8d..234790bf 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,4 +1,4 @@ -import { IEvent, IPayload } from "@clarity-types/data"; +import { IEventQueue, IPayload } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import * as decode from "@decode/clarity"; import config from "@src/core/config"; @@ -6,11 +6,12 @@ import {envelope} from "@src/data/metadata"; import { measure } from "@src/metric"; import metrics from "@src/metric/encode"; -export default function(events: IEvent[]): void { +export default function(events: IEventQueue): void { let payload: IPayload = { e: envelope(), m: metrics(), - d: events + s: events.server, + c: events.client }; let upload = config.upload ? config.upload : send; let data = JSON.stringify(payload); diff --git a/src/summary/index.ts b/src/summary/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/types/data.d.ts b/types/data.d.ts index 022c43dd..37d15083 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,5 +1,4 @@ import { IDecodedMetric } from "./metric"; -import { ISummary } from "./summary"; export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); @@ -28,6 +27,11 @@ export const enum Flush { None } +export interface IEventQueue { + server: IEvent[]; + client: IEvent[]; +} + export interface IEvent { t: number; e: Event; @@ -37,14 +41,13 @@ export interface IEvent { export interface IPayload { e: Token[]; m: Token[]; - s: Token[]; - d: IEvent[]; + s: IEvent[]; + c: IEvent[]; } export interface IDecodedPayload { envelope: IEnvelope; metrics: IDecodedMetric; - Summary: ISummary; data: IDecodedEvent[]; } diff --git a/types/summary.d.ts b/types/summary.d.ts deleted file mode 100644 index 584c5a91..00000000 --- a/types/summary.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export interface ISummary { - time: number; - duration: number; -} - -export interface IScrollSummary extends ISummary { - distanceX: number; - distanceY: number; -} From 800e71523f965644d662394744f9b8f3b0b14b3d Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sat, 17 Aug 2019 07:35:05 -0700 Subject: [PATCH 045/105] Renaming event queues --- src/data/queue.ts | 8 ++++---- src/data/upload.ts | 4 ++-- types/data.d.ts | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/data/queue.ts b/src/data/queue.ts index be46aee3..2c3cc59a 100644 --- a/src/data/queue.ts +++ b/src/data/queue.ts @@ -3,7 +3,7 @@ import config from "@src/core/config"; import upload from "@src/data/upload"; import recompute from "../core/recompute"; -let events: IEventQueue = { server: [], client: [] }; +let events: IEventQueue = { one: [], two: [] }; let timeout: number = null; window["PAYLOAD"] = []; @@ -20,10 +20,10 @@ export default function(timestamp: number, event: Event, data: Token[], flush: F case Event.Scroll: case Event.Document: case Event.Visibility: - events.server.push(e); + events.one.push(e); break; default: - events.client.push(e); + events.two.push(e); break; } @@ -46,5 +46,5 @@ function dequeue(): void { } function reset(): void { - events = { server: [], client: [] }; + events = { one: [], two: [] }; } diff --git a/src/data/upload.ts b/src/data/upload.ts index 234790bf..947a21c1 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -10,8 +10,8 @@ export default function(events: IEventQueue): void { let payload: IPayload = { e: envelope(), m: metrics(), - s: events.server, - c: events.client + a: events.one, + b: events.two }; let upload = config.upload ? config.upload : send; let data = JSON.stringify(payload); diff --git a/types/data.d.ts b/types/data.d.ts index 37d15083..09e0fdd3 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -28,8 +28,8 @@ export const enum Flush { } export interface IEventQueue { - server: IEvent[]; - client: IEvent[]; + one: IEvent[]; + two: IEvent[]; } export interface IEvent { @@ -41,8 +41,8 @@ export interface IEvent { export interface IPayload { e: Token[]; m: Token[]; - s: IEvent[]; - c: IEvent[]; + a: IEvent[]; + b: IEvent[]; } export interface IDecodedPayload { From 476600d0e577969f825c348fa2a60408a0ff77d5 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sat, 17 Aug 2019 09:14:44 -0700 Subject: [PATCH 046/105] Sending additive payload for metrics --- src/metric/encode.ts | 32 ++++++++++++++++++++++---------- src/metric/index.ts | 15 +++++++++++++++ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/metric/encode.ts b/src/metric/encode.ts index 2c1b9645..799f261c 100644 --- a/src/metric/encode.ts +++ b/src/metric/encode.ts @@ -1,6 +1,6 @@ import {Token} from "@clarity-types/data"; import { MetricType } from "@clarity-types/metric"; -import { metrics } from "@src/metric"; +import { metrics, reset, updates } from "@src/metric"; export default function(): Token[] { let output = []; @@ -10,8 +10,11 @@ export default function(): Token[] { let counters = metrics.counters; for (let metric in counters) { if (counters[metric]) { - output.push(parseInt(metric, 10)); - output.push(counters[metric]); + let m = num(metric); + if (updates.indexOf(m) >= 0) { + output.push(m); + output.push(counters[metric]); + } } } @@ -20,13 +23,16 @@ export default function(): Token[] { let summaries = metrics.measures; for (let metric in summaries) { if (summaries[metric]) { - let h = summaries[metric]; - output.push(parseInt(metric, 10)); - output.push(h.sum); - output.push(h.min); - output.push(h.max); - output.push(h.sumsquared); - output.push(h.count); + let m = num(metric); + if (updates.indexOf(m) >= 0) { + let h = summaries[metric]; + output.push(m); + output.push(h.sum); + output.push(h.min); + output.push(h.max); + output.push(h.sumsquared); + output.push(h.count); + } } } @@ -47,5 +53,11 @@ export default function(): Token[] { output.push(mark.time); } + reset(); + return output; } + +function num(input: string): number { + return parseInt(input, 10); +} diff --git a/src/metric/index.ts b/src/metric/index.ts index 077c40cd..513c5d7e 100644 --- a/src/metric/index.ts +++ b/src/metric/index.ts @@ -2,6 +2,7 @@ import { IMetric, Metric } from "@clarity-types/metric"; import time from "@src/core/time"; export let metrics: IMetric = null; +export let updates: Metric[] = []; export function start(): void { metrics = { counters: {}, measures: {}, events: [], marks: [] }; @@ -14,6 +15,7 @@ export function end(): void { export function counter(metric: Metric, increment: number = 1): void { if (!(metric in metrics.counters)) { metrics.counters[metric] = 0; } metrics.counters[metric] += increment; + track(metric); } export function measure(metric: Metric, value: number): void { @@ -23,6 +25,7 @@ export function measure(metric: Metric, value: number): void { metrics.measures[metric].max = metrics.measures[metric].max !== null ? Math.max(metrics.measures[metric].max, value) : value; metrics.measures[metric].sumsquared += (value * value); metrics.measures[metric].count++; + track(metric); } export function event(metric: Metric, begin: number, duration: number = 0): void { @@ -32,3 +35,15 @@ export function event(metric: Metric, begin: number, duration: number = 0): void export function mark(name: string): void { metrics.marks.push({ name, time: time() }); } + +function track(metric: Metric): void { + if (updates.indexOf(metric) === -1) { + updates.push(metric); + } +} + +export function reset(): void { + updates = []; + metrics.events = []; + metrics.marks = []; +} From e88b2a029c4102305284c9475faae1a6099eebe6 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 18 Aug 2019 22:08:15 -0700 Subject: [PATCH 047/105] Optimizing encoding --- decode/clarity.ts | 37 +++++++++++++++++++++---------------- decode/dom.ts | 12 +++++++----- decode/metadata.ts | 23 ++++++++++++++--------- decode/render.ts | 8 +++++--- decode/viewport.ts | 26 ++++++++++++++------------ src/data/encode.ts | 12 ++++++++++-- src/data/metadata.ts | 12 ++++-------- src/data/queue.ts | 8 ++++---- src/diagnostic/encode.ts | 3 ++- src/diagnostic/image.ts | 3 +-- src/diagnostic/script.ts | 3 +-- src/dom/discover.ts | 5 ++--- src/dom/encode.ts | 22 ++++++++++++---------- src/dom/mutation.ts | 9 ++------- src/interaction/encode.ts | 19 +++++++++++++++---- src/interaction/mouse.ts | 6 ++---- src/viewport/document.ts | 3 +-- src/viewport/encode.ts | 11 +++++++++-- src/viewport/resize.ts | 4 ++-- src/viewport/scroll.ts | 18 +++++++----------- src/viewport/visibility.ts | 3 +-- types/data.d.ts | 17 ++++++----------- types/interaction.d.ts | 2 +- 23 files changed, 143 insertions(+), 123 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 6f3318d2..269814fb 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,4 +1,4 @@ -import { Event, IDecodedEvent, IDecodedPayload, IEvent, IPayload } from "../types/data"; +import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; import dom from "./dom"; import envelope from "./envelope"; import metadata from "./metadata"; @@ -14,7 +14,7 @@ export function json(data: string): IDecodedPayload { let decoded: IDecodedPayload = { envelope: envelope(payload.e), metrics: metric(payload.m), - data: null + events: [] }; if (pageId !== decoded.envelope.pageId) { @@ -23,30 +23,30 @@ export function json(data: string): IDecodedPayload { r.reset(); } - let events: IDecodedEvent[] = []; - let encoded: IEvent[] = merge(payload.s, payload.c); + let encoded: Token[][] = merge(payload.a, payload.b); payloads.push(payload); for (let entry of encoded) { - let exploded: IDecodedEvent = { time: entry.t, event: entry.e, data: null }; - switch (entry.e) { + let event: IDecodedEvent; + switch (entry[1]) { case Event.Scroll: case Event.Document: case Event.Resize: - exploded.data = viewport(entry.d, entry.e); + event = viewport(entry); break; case Event.Discover: case Event.Mutation: - exploded.data = dom(entry.d, entry.e); + event = dom(entry); break; case Event.Metadata: - exploded.data = metadata(entry.d, entry.e); + event = metadata(entry); + break; + default: + event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; break; } - events.push(exploded); + decoded.events.push(event); } - decoded.data = events; - return decoded; } @@ -58,12 +58,13 @@ export function html(data: string): string { export function render(data: string, iframe: HTMLIFrameElement, header?: HTMLElement): void { let decoded = json(data); + console.log("Decoded: " + JSON.stringify(decoded)); // Render metrics r.metrics(decoded.metrics, header); // Render events - let events = decoded.data; + let events = decoded.events; for (let entry of events) { switch (entry.event) { case Event.Discover: @@ -74,12 +75,16 @@ export function render(data: string, iframe: HTMLIFrameElement, header?: HTMLEle r.resize(entry.data[0], iframe); break; case Event.Scroll: - r.scroll(entry.data[0], iframe); + r.scroll(entry.data, iframe); break; } } } -function merge(one: IEvent[], two: IEvent[]): IEvent[] { - return one.concat(two).sort(function(a: IEvent, b: IEvent): number { return a.t - b.t; }); +function merge(one: Token[][], two: Token[][]): Token[][] { + return one.concat(two).sort(function(a: Token[], b: Token[]): number { + let x = a[0] as number; + let y = b[0] as number; + return x - y; + }); } diff --git a/decode/dom.ts b/decode/dom.ts index 6e40a7b0..35760b50 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -1,19 +1,21 @@ import { resolve } from "../src/data/token"; -import { Event, Token } from "../types/data"; +import { Event, IDecodedEvent, Token } from "../types/data"; import { IDecodedNode } from "../types/dom"; -export default function(tokens: Token[], event: Event): IDecodedNode[] { +export default function(tokens: Token[]): IDecodedEvent { let number = 0; let lastType = null; let node = []; - let decoded: IDecodedNode[] = []; + let time = tokens[0] as number; + let event = tokens[1] as Event; + let decoded: IDecodedEvent = {time, event, data: []}; let tagIndex = 0; for (let token of tokens) { let type = typeof(token); switch (type) { case "number": if (type !== lastType && lastType !== null) { - decoded.push(process(node, tagIndex)); + decoded.data.push(process(node, tagIndex)); node = []; tagIndex = 0; } @@ -45,7 +47,7 @@ export default function(tokens: Token[], event: Event): IDecodedNode[] { } // Process last node - decoded.push(process(node, tagIndex)); + decoded.data.push(process(node, tagIndex)); return decoded; } diff --git a/decode/metadata.ts b/decode/metadata.ts index 2d6ce94f..20f07af7 100644 --- a/decode/metadata.ts +++ b/decode/metadata.ts @@ -1,13 +1,18 @@ -import { Event, IMetadata, Token } from "../types/data"; +import { Event, IDecodedEvent, Token } from "../types/data"; -export default function(tokens: Token[], event: Event): IMetadata { +export default function(tokens: Token[]): IDecodedEvent { return { - sequence: tokens[0] as number, - version: tokens[1] as string, - pageId: tokens[2] as string, - userId: tokens[3] as string, - projectId: tokens[4] as string, - url: tokens[5] as string, - title: tokens[6] as string + time: tokens[0] as number, + event: tokens[1] as Event, + data: { + sequence: tokens[2] as number, + version: tokens[3] as string, + pageId: tokens[4] as string, + userId: tokens[5] as string, + projectId: tokens[6] as string, + url: tokens[7] as string, + title: tokens[8] as string, + referrer: tokens[9] as string + } }; } diff --git a/decode/render.ts b/decode/render.ts index 5e21a40c..b3059a79 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -140,7 +140,7 @@ function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void try { parent.insertBefore(node, next); } catch (ex) { - console.error("Node: " + node + " | Parent: " + parent); + console.error("Node: " + node + " | Parent: " + parent + " | Data: " + JSON.stringify(data)); console.error("Exception encountered while inserting node: " + ex); } } else if (parent === null && node.parentElement !== null) { @@ -172,8 +172,10 @@ function setAttributes(node: HTMLElement, attributes: object): void { } } -export function scroll(data: IScrollViewport, placeholder: HTMLIFrameElement): void { - placeholder.contentWindow.scrollTo(data.x, data.y); +export function scroll(data: IScrollViewport[], placeholder: HTMLIFrameElement): void { + for (let d of data) { + placeholder.contentWindow.scrollTo(d.x, d.y); + } } export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): void { diff --git a/decode/viewport.ts b/decode/viewport.ts index fea1f50d..f314c5a4 100644 --- a/decode/viewport.ts +++ b/decode/viewport.ts @@ -1,22 +1,24 @@ -import { DecodedToken, Event, Token } from "../types/data"; +import { Event, IDecodedEvent, Token } from "../types/data"; import { IDocumentSize, IResizeViewport, IScrollViewport } from "../types/viewport"; -export default function(tokens: Token[], event: Event): DecodedToken[] { - let decoded: DecodedToken[] = []; +export default function(tokens: Token[]): IDecodedEvent { + let time = tokens[0] as number; + let event = tokens[1] as Event; + let decoded: IDecodedEvent = {time, event, data: []}; switch (event) { case Event.Resize: - let r: IResizeViewport = { width: tokens[0] as number, height: tokens[1] as number }; - decoded.push(r); + let r: IResizeViewport = { width: tokens[2] as number, height: tokens[3] as number }; + decoded.data.push(r); case Event.Document: - let d: IDocumentSize = { width: tokens[0] as number, height: tokens[1] as number }; - decoded.push(d); + let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; + decoded.data.push(d); break; case Event.Scroll: - let time = 0; - for (let i = 0; i < tokens.length; i = i + 3) { - time += tokens[i] as number; - let s: IScrollViewport = { time, x: tokens[i + 1] as number, y: tokens[i + 2] as number }; - decoded.push(s); + let t = time; + for (let i = 2; i < tokens.length; i = i + 3) { + t += tokens[i] as number; + let s: IScrollViewport = { time: t, x: tokens[i + 1] as number, y: tokens[i + 2] as number }; + decoded.data.push(s); } break; } diff --git a/src/data/encode.ts b/src/data/encode.ts index c0edfd08..b7414fc1 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -1,8 +1,15 @@ -import {Token} from "@clarity-types/data"; +import {Event, Token} from "@clarity-types/data"; +import { Metric } from "@clarity-types/metric"; +import time from "@src/core/time"; import { metadata } from "@src/data/metadata"; +import * as metric from "@src/metric"; export default function(envelope: boolean = false): Token[] { - let tokens = []; + let t = time(); + let tokens: Token[] = envelope ? [] : [t, Event.Metadata]; + + metric.measure(Metric.WireupLag, t); + tokens.push(metadata.sequence); tokens.push(metadata.version); tokens.push(metadata.pageId); @@ -11,6 +18,7 @@ export default function(envelope: boolean = false): Token[] { if (envelope === false) { tokens.push(metadata.url); tokens.push(metadata.title); + tokens.push(metadata.referrer); } return tokens; } diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 29ff25b3..34466565 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,17 +1,12 @@ -import { Event, Flush, IMetadata, Token } from "@clarity-types/data"; -import { Metric } from "@clarity-types/metric"; -import time from "@src/core/time"; +import { Flush, IMetadata, Token } from "@clarity-types/data"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; import queue from "@src/data/queue"; -import * as metric from "@src/metric"; export let metadata: IMetadata = null; export function start(): void { - metric.measure(Metric.WireupLag, time()); - metadata = { sequence: 0, version, @@ -19,10 +14,11 @@ export function start(): void { userId: guid(), projectId: hash(location.host), url: location.href, - title: document.title + title: document.title, + referrer: document.referrer }; - queue(time(), Event.Metadata, encode(), Flush.None); + queue(encode(), Flush.None); } export function end(): void { diff --git a/src/data/queue.ts b/src/data/queue.ts index 2c3cc59a..ff9a0ccc 100644 --- a/src/data/queue.ts +++ b/src/data/queue.ts @@ -8,8 +8,8 @@ let timeout: number = null; window["PAYLOAD"] = []; -export default function(timestamp: number, event: Event, data: Token[], flush: Flush = Flush.Schedule): void { - let e = {t: timestamp, e: event, d: data}; +export default function(data: Token[], flush: Flush = Flush.Schedule): void { + let event = data[1]; switch (event) { case Event.Mouse: @@ -20,10 +20,10 @@ export default function(timestamp: number, event: Event, data: Token[], flush: F case Event.Scroll: case Event.Document: case Event.Visibility: - events.one.push(e); + events.one.push(data); break; default: - events.two.push(e); + events.two.push(data); break; } diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts index 80b70f9f..dcc5f1fa 100644 --- a/src/diagnostic/encode.ts +++ b/src/diagnostic/encode.ts @@ -1,11 +1,12 @@ import {Event, Token} from "@clarity-types/data"; import {Metric} from "@clarity-types/metric"; +import time from "@src/core/time"; import * as image from "@src/diagnostic/image"; import * as script from "@src/diagnostic/script"; import * as metric from "@src/metric"; export default function(type: Event): Token[] { - let tokens = []; + let tokens: Token[] = [time(), type]; switch (type) { case Event.ScriptError: diff --git a/src/diagnostic/image.ts b/src/diagnostic/image.ts index ededd879..2804600e 100644 --- a/src/diagnostic/image.ts +++ b/src/diagnostic/image.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; import { IImageError } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; -import time from "@src/core/time"; import queue from "@src/data/queue"; import { getId } from "@src/dom/virtualdom"; import encode from "./encode"; @@ -25,7 +24,7 @@ function handler(error: ErrorEvent): void { }); } - queue(time(), Event.ImageError, encode(Event.ImageError)); + queue(encode(Event.ImageError)); } export function reset(): void { diff --git a/src/diagnostic/script.ts b/src/diagnostic/script.ts index c56e6945..4031b2e3 100644 --- a/src/diagnostic/script.ts +++ b/src/diagnostic/script.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; import { IScriptError } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; -import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "./encode"; @@ -26,7 +25,7 @@ function handler(error: ErrorEvent): void { source: error["filename"] }); - queue(time(), Event.ScriptError, encode(Event.ScriptError)); + queue(encode(Event.ScriptError)); } export function reset(): void { diff --git a/src/dom/discover.ts b/src/dom/discover.ts index f61b0643..1a355d3c 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -2,7 +2,6 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; -import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "@src/dom/encode"; @@ -10,7 +9,7 @@ import processNode from "./node"; export function start(): void { discover().then((data: Token[]) => { - queue(time(), Event.Discover, data); + queue(data); }); } @@ -24,7 +23,7 @@ async function discover(): Promise { processNode(node, Source.Discover); node = walker.nextNode(); } - let data = await encode(timer); + let data = await encode(Event.Discover); task.stop(timer); return data; } diff --git a/src/dom/encode.ts b/src/dom/encode.ts index ccf8511a..66b0790b 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -1,7 +1,8 @@ -import {Token} from "@clarity-types/data"; +import {Event, Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; +import time from "@src/core/time"; import hash from "@src/data/hash"; import {check} from "@src/data/token"; import * as metrics from "@src/metric"; @@ -11,8 +12,9 @@ import * as nodes from "./virtualdom"; window["HASH"] = hash; let reference: number = 0; -export default async function(timer: Metric): Promise { - let markup = []; +export default async function(type: Event): Promise { + let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; + let tokens: Token[] = [time(), type]; let values = nodes.summarize(); reference = 0; for (let value of values) { @@ -26,9 +28,9 @@ export default async function(timer: Metric): Promise { switch (key) { case "tag": metrics.counter(Metric.NodeCount); - markup.push(number(value.id)); - if (value.parent) { markup.push(number(value.parent)); } - if (value.next) { markup.push(number(value.next)); } + tokens.push(number(value.id)); + if (value.parent) { tokens.push(number(value.parent)); } + if (value.next) { tokens.push(number(value.next)); } metadata.push(data[key]); break; case "attributes": @@ -61,15 +63,15 @@ export default async function(timer: Metric): Promise { // Add metadata metadata = meta(metadata); for (let token of metadata) { - let index: number = typeof token === "string" ? markup.indexOf(token) : -1; - markup.push(index >= 0 && token.length > index.toString().length ? [index] : token); + let index: number = typeof token === "string" ? tokens.indexOf(token) : -1; + tokens.push(index >= 0 && token.length > index.toString().length ? [index] : token); } // Add layout boxes for (let entry of layouts) { - markup.push(entry); + tokens.push(entry); } } - return markup; + return tokens; } function meta(metadata: string[]): string[] | string[][] { diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index be4e9e26..dc6fcf19 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -2,16 +2,13 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/dom"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; -import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "@src/dom/encode"; import processNode from "./node"; let observer: MutationObserver; -window["MUTATIONS"] = []; export function start(): void { - console.log("Listening for mutations..."); if (observer) { observer.disconnect(); } @@ -25,10 +22,8 @@ export function end(): void { } function handle(mutations: MutationRecord[]): void { - window["MUTATIONS"].push(time()); - window["MUTATIONS"].push(mutations); process(mutations).then((data: Token[]) => { - queue(time(), Event.Mutation, data); + queue(data); }); } @@ -72,7 +67,7 @@ async function process(mutations: MutationRecord[]): Promise { break; } } - let data = await encode(timer); + let data = await encode(Event.Mutation); task.stop(timer); return data; } diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index e13e55ac..a18303be 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -1,22 +1,33 @@ import {Event, Token} from "@clarity-types/data"; +import { Mouse } from "@clarity-types/interaction"; import * as mouse from "./mouse"; export default function(type: Event): Token[] { - let tokens = []; + let tokens: Token[] = []; switch (type) { case Event.Mouse: let m = mouse.summarize(); let timestamp: number = null; + let mouseType: Mouse = null; for (let i = 0; i < m.length; i++) { let entry = m[i]; - timestamp = (i === 0) ? entry.time : timestamp; + + if (i === 0) { + timestamp = entry.time; + tokens.push(timestamp); + tokens.push(type); + } + + if (mouseType !== entry.type) { tokens.push(entry.type); } tokens.push(entry.time - timestamp); - tokens.push(entry.type); tokens.push(entry.x); tokens.push(entry.y); tokens.push(entry.target); - if (entry.buttons > 0) { tokens.push(entry.buttons); } + tokens.push(entry.buttons); + + timestamp = entry.time; + mouseType = entry.type; } mouse.reset(); break; diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index f1da8fdc..a1a870ff 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -9,7 +9,6 @@ import encode from "./encode"; let data: IMouseInteraction[] = []; let timeout: number = null; -let timestamp: number = null; export function start(): void { bind(document, "mousedown", handler.bind(this, Mouse.Down)); @@ -22,8 +21,8 @@ export function start(): void { function handler(type: Mouse, evt: MouseEvent): void { let de = document.documentElement; data.push({ - time: time(), type, + time: time(), x: "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null), y: "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null), target: evt.target ? getId(evt.target as Node) : null, @@ -34,7 +33,7 @@ function handler(type: Mouse, evt: MouseEvent): void { } function schedule(): void { - queue(timestamp, Event.Mouse, encode(Event.Mouse)); + queue(encode(Event.Mouse)); } export function reset(): void { @@ -50,7 +49,6 @@ export function summarize(): IMouseInteraction[] { if (isFirst || index === data.length - 1 || checkDistance(last, entry)) { - timestamp = isFirst ? entry.time : timestamp; summary.push(entry); } index++; diff --git a/src/viewport/document.ts b/src/viewport/document.ts index c642a411..a14038d4 100644 --- a/src/viewport/document.ts +++ b/src/viewport/document.ts @@ -1,6 +1,5 @@ import { Event, Flush } from "@clarity-types/data"; import { IDocumentSize } from "@clarity-types/viewport"; -import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "./encode"; @@ -27,7 +26,7 @@ function recompute(): void { height: documentHeight }; - queue(time(), Event.Document, encode(Event.Document), Flush.None); + queue(encode(Event.Document), Flush.None); } export function compute(): void { diff --git a/src/viewport/encode.ts b/src/viewport/encode.ts index 68351906..52b8008f 100644 --- a/src/viewport/encode.ts +++ b/src/viewport/encode.ts @@ -1,5 +1,6 @@ import {Event, Token} from "@clarity-types/data"; import {Metric} from "@clarity-types/metric"; +import time from "@src/core/time"; import * as metric from "@src/metric"; import * as document from "./document"; import * as resize from "./resize"; @@ -7,7 +8,7 @@ import * as scroll from "./scroll"; import * as visibility from "./visibility"; export default function(type: Event): Token[] { - let tokens = []; + let tokens: Token[] = [time(), type]; switch (type) { case Event.Resize: @@ -31,10 +32,16 @@ export default function(type: Event): Token[] { let timestamp: number = null; for (let i = 0; i < s.length; i++) { let entry = s[i]; - timestamp = (i === 0) ? entry.time : timestamp; + + if (i === 0) { + timestamp = entry.time; + tokens[0] = timestamp; + } + tokens.push(entry.time - timestamp); tokens.push(entry.x); tokens.push(entry.y); + timestamp = entry.time; } scroll.reset(); break; diff --git a/src/viewport/resize.ts b/src/viewport/resize.ts index db1c5f8e..ddcd0ba8 100644 --- a/src/viewport/resize.ts +++ b/src/viewport/resize.ts @@ -1,7 +1,7 @@ import { Event } from "@clarity-types/data"; import { IResizeViewport } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; -import time from "@src/core/time"; + import queue from "@src/data/queue"; import encode from "./encode"; @@ -17,7 +17,7 @@ function recompute(): void { width: "innerWidth" in window ? window.innerWidth : document.documentElement.clientWidth, height: "innerHeight" in window ? window.innerHeight : document.documentElement.clientHeight }; - queue(time(), Event.Resize, encode(Event.Resize)); + queue(encode(Event.Resize)); } export function reset(): void { diff --git a/src/viewport/scroll.ts b/src/viewport/scroll.ts index 49c2b777..1a241ee2 100644 --- a/src/viewport/scroll.ts +++ b/src/viewport/scroll.ts @@ -8,7 +8,6 @@ import encode from "./encode"; let data: IScrollViewport[] = []; let timeout: number = null; -let timestamp: number = null; export function start(): void { bind(window, "scroll", recompute); @@ -24,7 +23,7 @@ function recompute(): void { } function schedule(): void { - queue(timestamp, Event.Scroll, encode(Event.Scroll)); + queue(encode(Event.Scroll)); } export function reset(): void { @@ -33,18 +32,15 @@ export function reset(): void { export function summarize(): IScrollViewport[] { let summary: IScrollViewport[] = []; - let index = 0; let last = null; - for (let entry of data) { - let isFirst = index === 0; - if (isFirst - || index === data.length - 1 - || checkDistance(last, entry)) { - timestamp = isFirst ? entry.time : timestamp; + for (let i = 0; i < data.length; i++) { + let entry = data[i]; + let isFirst = i === 0; + let isLast = i === data.length - 1; + if (isFirst || isLast || checkDistance(last, entry)) { summary.push(entry); + last = entry; } - index++; - last = entry; } return summary; } diff --git a/src/viewport/visibility.ts b/src/viewport/visibility.ts index 96ee347a..c044a5ac 100644 --- a/src/viewport/visibility.ts +++ b/src/viewport/visibility.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; import { IPageVisibility } from "@clarity-types/viewport"; import { bind } from "@src/core/event"; -import time from "@src/core/time"; import queue from "@src/data/queue"; import encode from "./encode"; @@ -16,7 +15,7 @@ export function start(): void { function recompute(): void { data = { visible: "visibilityState" in document ? document.visibilityState : "default" }; - queue(time(), Event.Visibility, encode(Event.Visibility)); + queue(encode(Event.Visibility)); } export function reset(): void { diff --git a/types/data.d.ts b/types/data.d.ts index 09e0fdd3..4805f23b 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -28,27 +28,21 @@ export const enum Flush { } export interface IEventQueue { - one: IEvent[]; - two: IEvent[]; -} - -export interface IEvent { - t: number; - e: Event; - d: Token[]; + one: Token[][]; + two: Token[][]; } export interface IPayload { e: Token[]; m: Token[]; - a: IEvent[]; - b: IEvent[]; + a: Token[][]; + b: Token[][]; } export interface IDecodedPayload { envelope: IEnvelope; metrics: IDecodedMetric; - data: IDecodedEvent[]; + events: IDecodedEvent[]; } export interface IDecodedEvent { @@ -68,4 +62,5 @@ export interface IEnvelope { export interface IMetadata extends IEnvelope { url: string; title: string; + referrer: string; } diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 25f2cc30..61fef632 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -7,8 +7,8 @@ export const enum Mouse { } interface IMouseInteraction { - time: number; type: Mouse; + time: number; x: number; y: number; target: number; From b96a510c6c5fa3a4d9c88f792c25f45829ff372c Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 19 Aug 2019 16:23:41 -0700 Subject: [PATCH 048/105] Code refactor and adding index for each module --- src/clarity.ts | 52 +++++++++++++--------------------------- src/core/index.ts | 16 +++++++++++++ src/data/index.ts | 9 +++++++ src/dom/index.ts | 14 +++++++++++ src/dom/virtualdom.ts | 9 +++++++ src/interaction/index.ts | 9 +++++++ src/viewport/index.ts | 15 ++++++++++++ 7 files changed, 89 insertions(+), 35 deletions(-) create mode 100644 src/core/index.ts create mode 100644 src/data/index.ts create mode 100644 src/dom/index.ts create mode 100644 src/interaction/index.ts create mode 100644 src/viewport/index.ts diff --git a/src/clarity.ts b/src/clarity.ts index 6757ad30..a96ac783 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,55 +1,37 @@ import { IConfig } from "@clarity-types/core"; -import config from "@src/core/config"; -import * as event from "@src/core/event"; -import * as metadata from "@src/data/metadata"; +import * as core from "@src/core"; +import * as data from "@src/data"; import * as diagnostic from "@src/diagnostic"; -import * as discover from "@src/dom/discover"; -import * as mutation from "@src/dom/mutation"; -import * as mouse from "@src/interaction/mouse"; +import * as dom from "@src/dom"; +import * as interaction from "@src/interaction"; import * as metric from "@src/metric"; -import * as document from "@src/viewport/document"; -import * as resize from "@src/viewport/resize"; -import * as scroll from "@src/viewport/scroll"; -import * as visibility from "@src/viewport/visibility"; +import * as viewport from "@src/viewport"; let status = false; /* Initial discovery of DOM */ export function start(configuration: IConfig = {}): void { - - // Process custom configuration, if available - for (let key in configuration) { - if (key in config) { config[key] = configuration[key]; } - } - - event.reset(); + core.start(configuration); metric.start(); - metadata.start(); + data.start(); diagnostic.start(); - - // DOM - mutation.start(); - discover.start(); - - // Viewport - document.start(); - resize.start(); - visibility.start(); - scroll.start(); - - // Pointer - mouse.start(); + dom.start(); + viewport.start(); + interaction.start(); // Mark Clarity session as active status = true; } export function end(): void { - event.reset(); - metadata.end(); - mutation.end(); - metric.end(); + interaction.end(); + viewport.end(); + dom.end(); diagnostic.end(); + data.end(); + metric.end(); + core.end(); + status = false; } diff --git a/src/core/index.ts b/src/core/index.ts new file mode 100644 index 00000000..a50bc9a8 --- /dev/null +++ b/src/core/index.ts @@ -0,0 +1,16 @@ +import { IConfig } from "@clarity-types/core"; +import config from "@src/core/config"; +import * as event from "@src/core/event"; + +export function start(configuration: IConfig): void { + // Process custom configuration, if available + for (let key in configuration) { + if (key in config) { config[key] = configuration[key]; } + } + + event.reset(); +} + +export function end(): void { + event.reset(); +} diff --git a/src/data/index.ts b/src/data/index.ts new file mode 100644 index 00000000..11ee459c --- /dev/null +++ b/src/data/index.ts @@ -0,0 +1,9 @@ +import * as metadata from "@src/data/metadata"; + +export function start(): void { + metadata.start(); +} + +export function end(): void { + metadata.end(); +} diff --git a/src/dom/index.ts b/src/dom/index.ts new file mode 100644 index 00000000..6a4155dc --- /dev/null +++ b/src/dom/index.ts @@ -0,0 +1,14 @@ +import * as discover from "@src/dom/discover"; +import * as mutation from "@src/dom/mutation"; +import * as virtualdom from "@src/dom/virtualdom"; + +export function start(): void { + virtualdom.reset(); + mutation.start(); + discover.start(); +} + +export function end(): void { + virtualdom.reset(); + mutation.end(); +} diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 4a708a31..82391dc5 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -16,6 +16,15 @@ let backupValues: INodeValue[]; // For debugging window["DOM"] = { getId, get, getNode, changes }; +export function reset(): void { + index = 1; + nodes = []; + values = []; + updates = []; + changes = []; + console.log("Window Random: " + window["RANDOM"]); +} + export function getId(node: Node, autogen: boolean = false): number { if (node === null) { return null; } let id = node[NODE_ID_PROP]; diff --git a/src/interaction/index.ts b/src/interaction/index.ts new file mode 100644 index 00000000..1dc5d8ad --- /dev/null +++ b/src/interaction/index.ts @@ -0,0 +1,9 @@ +import * as mouse from "@src/interaction/mouse"; + +export function start(): void { + mouse.start(); +} + +export function end(): void { + // End calls +} diff --git a/src/viewport/index.ts b/src/viewport/index.ts new file mode 100644 index 00000000..d96aee41 --- /dev/null +++ b/src/viewport/index.ts @@ -0,0 +1,15 @@ +import * as document from "@src/viewport/document"; +import * as resize from "@src/viewport/resize"; +import * as scroll from "@src/viewport/scroll"; +import * as visibility from "@src/viewport/visibility"; + +export function start(): void { + document.start(); + resize.start(); + visibility.start(); + scroll.start(); +} + +export function end(): void { + // End calls +} From 8bf9f37544a237fdd867c7dd561405d5430fe9ce Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 19 Aug 2019 19:06:29 -0700 Subject: [PATCH 049/105] Bug fixes to account for new encoding --- decode/dom.ts | 5 +++-- decode/metric.ts | 4 ++-- decode/render.ts | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/decode/dom.ts b/decode/dom.ts index 35760b50..8d37c5fb 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -10,7 +10,8 @@ export default function(tokens: Token[]): IDecodedEvent { let event = tokens[1] as Event; let decoded: IDecodedEvent = {time, event, data: []}; let tagIndex = 0; - for (let token of tokens) { + for (let i = 2; i < tokens.length; i++) { + let token = tokens[i]; let type = typeof(token); switch (type) { case "number": @@ -63,7 +64,7 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let layouts = []; let attributes = {}; let value = null; - + console.log("Node: " + JSON.stringify(node) + " | Tag: " + tagIndex); for (let i = tagIndex + 1; i < node.length; i++) { let token = node[i] as string; let keyIndex = token.indexOf("="); diff --git a/decode/metric.ts b/decode/metric.ts index bec8e95f..12441e4f 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -21,9 +21,9 @@ map[Metric.DocumentHeight] = { name: "Document Height", unit: "px", value: "max" map[Metric.ClickEvent] = { name: "Click Event", unit: ""}; map[Metric.InteractionEvent] = { name: "Interaction Event", unit: ""}; -export default function(tokens: Token[]): IDecodedMetric { - let metrics: IDecodedMetric = { counters: {}, measures: {}, events: [], marks: [], map }; +let metrics: IDecodedMetric = { counters: {}, measures: {}, events: [], marks: [], map }; +export default function(tokens: Token[]): IDecodedMetric { let i = 0; let metricType = null; while (i < tokens.length) { diff --git a/decode/render.ts b/decode/render.ts index b3059a79..b12685f3 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -140,8 +140,8 @@ function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void try { parent.insertBefore(node, next); } catch (ex) { - console.error("Node: " + node + " | Parent: " + parent + " | Data: " + JSON.stringify(data)); - console.error("Exception encountered while inserting node: " + ex); + console.warn("Node: " + node + " | Parent: " + parent + " | Data: " + JSON.stringify(data)); + console.warn("Exception encountered while inserting node: " + ex); } } else if (parent === null && node.parentElement !== null) { node.parentElement.removeChild(node); @@ -165,8 +165,8 @@ function setAttributes(node: HTMLElement, attributes: object): void { try { node.setAttribute(attribute, attributes[attribute]); } catch (ex) { - console.error("Node: " + node + " | " + JSON.stringify(attributes)); - console.error("Exception encountered while adding attributes: " + ex); + console.warn("Node: " + node + " | " + JSON.stringify(attributes)); + console.warn("Exception encountered while adding attributes: " + ex); } } } From 7ada9a6d07a6c0acfe5bc59b5950d6757c8db240 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 19 Aug 2019 20:40:31 -0700 Subject: [PATCH 050/105] Getting rid of delta incrementing numbers --- decode/dom.ts | 14 +++++++------- src/dom/encode.ts | 17 +++-------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/decode/dom.ts b/decode/dom.ts index 8d37c5fb..52f7738e 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -3,7 +3,6 @@ import { Event, IDecodedEvent, Token } from "../types/data"; import { IDecodedNode } from "../types/dom"; export default function(tokens: Token[]): IDecodedEvent { - let number = 0; let lastType = null; let node = []; let time = tokens[0] as number; @@ -20,8 +19,6 @@ export default function(tokens: Token[]): IDecodedEvent { node = []; tagIndex = 0; } - number += token as number; - token = number; node.push(token); tagIndex++; break; @@ -80,12 +77,15 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { layout.push(parseInt(part, 36)); } layouts.push(layout); - } else if (output.tag === "*T" && parts.length === 2 && parts[0].length > 0) { - let textCount = parseInt(parts[0], 36); - let wordCount = parseInt(parts[1], 36); - value = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); } else if (output.tag === "*T") { value = token; + if (parts.length === 2) { + let textCount = parseInt(parts[0], 36); + let wordCount = parseInt(parts[1], 36); + if (isFinite(textCount) && isFinite(wordCount)) { + value = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); + } + } } } diff --git a/src/dom/encode.ts b/src/dom/encode.ts index 66b0790b..b7d9a85f 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -10,13 +10,11 @@ import * as metrics from "@src/metric"; import * as nodes from "./virtualdom"; window["HASH"] = hash; -let reference: number = 0; export default async function(type: Event): Promise { let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; let tokens: Token[] = [time(), type]; let values = nodes.summarize(); - reference = 0; for (let value of values) { if (task.longtask(timer)) { await task.idle(timer); } let metadata = []; @@ -28,9 +26,9 @@ export default async function(type: Event): Promise { switch (key) { case "tag": metrics.counter(Metric.NodeCount); - tokens.push(number(value.id)); - if (value.parent) { tokens.push(number(value.parent)); } - if (value.next) { tokens.push(number(value.next)); } + tokens.push(value.id); + if (value.parent) { tokens.push(value.parent); } + if (value.next) { tokens.push(value.next); } metadata.push(data[key]); break; case "attributes": @@ -108,12 +106,3 @@ function layout(l: number[]): string[] { } return output; } - -function number(id: number): number { - let output = id; - if (id > 0) { - output = id - reference; - reference = id; - } - return output; -} From a7e7feb125f510c804f55a3db1f45d1640bb6c0b Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 20 Aug 2019 08:05:44 -0700 Subject: [PATCH 051/105] Adding devtools support & masking text by default --- decode/dom.ts | 2 +- src/core/config.ts | 1 + src/dom/encode.ts | 10 ++++----- src/dom/virtualdom.ts | 51 ++++++++++++++++++++----------------------- types/core.d.ts | 1 + 5 files changed, 32 insertions(+), 33 deletions(-) diff --git a/decode/dom.ts b/decode/dom.ts index 52f7738e..28e98f5d 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -83,7 +83,7 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let textCount = parseInt(parts[0], 36); let wordCount = parseInt(parts[1], 36); if (isFinite(textCount) && isFinite(wordCount)) { - value = wordCount > 0 && textCount === 0 ? " " : Array((textCount + 1) / 2).join("* "); + value = wordCount > 0 && textCount === 0 ? " " : Array(Math.floor((textCount + 1) / 2)).join("* "); } } } diff --git a/src/core/config.ts b/src/core/config.ts index 4cdd80ad..93b8ba09 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -5,6 +5,7 @@ let config: IConfig = { lookahead: 250, distance: 20, delay: 250, + showText: false, cssRules: false, tokens: [], upload: null diff --git a/src/dom/encode.ts b/src/dom/encode.ts index b7d9a85f..e9dac9ec 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -1,6 +1,7 @@ import {Event, Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; import { Metric } from "@clarity-types/metric"; +import config from "@src/core/config"; import * as task from "@src/core/task"; import time from "@src/core/time"; import hash from "@src/data/hash"; @@ -48,11 +49,10 @@ export default async function(type: Event): Promise { break; case "value": let parent = nodes.getNode(value.parent); - if (parent === null) { - console.warn("Unexpected | Node data: " + JSON.stringify(data)); - } + if (parent === null) { console.warn("Unexpected | Node value: " + JSON.stringify(value)); } let parentTag = nodes.get(parent) ? nodes.get(parent).data.tag : null; - metadata.push(text(parentTag, data[key])); + let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; + metadata.push(text(tag, data[key])); break; } } @@ -84,7 +84,7 @@ function text(tag: string, value: string): string { case "TITLE": return value; default: - return value; + if (config.showText) { return value; } let wasWhiteSpace = false; let textCount = 0; let wordCount = 0; diff --git a/src/dom/virtualdom.ts b/src/dom/virtualdom.ts index 82391dc5..75b05625 100644 --- a/src/dom/virtualdom.ts +++ b/src/dom/virtualdom.ts @@ -2,6 +2,7 @@ import { INodeChange, INodeData, INodeValue, Source } from "@clarity-types/dom"; import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; +const DEVTOOLS_HOOK: string = "__CLARITY_DEVTOOLS_HOOK__"; let index: number = 1; let nodes: Node[] = []; @@ -13,16 +14,13 @@ let backupIndex: number; let backupNodes: Node[]; let backupValues: INodeValue[]; -// For debugging -window["DOM"] = { getId, get, getNode, changes }; - export function reset(): void { index = 1; nodes = []; values = []; updates = []; changes = []; - console.log("Window Random: " + window["RANDOM"]); + if (DEVTOOLS_HOOK in window) { window[DEVTOOLS_HOOK] = { get, getNode, history }; } } export function getId(node: Node, autogen: boolean = false): number { @@ -102,15 +100,6 @@ export function update(node: Node, data: INodeData, source: Source): void { } } -function getNextId(node: Node): number { - let id = null; - while (id === null && node.nextSibling) { - id = getId(node.nextSibling); - node = node.nextSibling; - } - return id; -} - export function getNode(id: number): Node { if (id in nodes) { return nodes[id]; @@ -127,16 +116,6 @@ export function has(node: Node): boolean { return getId(node) in nodes; } -export function getNodes(): Node[] { - let n: Node[] = []; - for (let id in nodes) { - if (nodes[id]) { - n.push(nodes[id]); - } - } - return n; -} - export function summarize(): INodeValue[] { let v = []; for (let id of updates) { @@ -160,14 +139,32 @@ export function rollback(): void { index = backupIndex; } +function getNextId(node: Node): number { + let id = null; + while (id === null && node.nextSibling) { + id = getId(node.nextSibling); + node = node.nextSibling; + } + return id; +} + function copy(input: INodeValue[]): INodeValue[] { return JSON.parse(JSON.stringify(input)); } function track(id: number, source: Source): void { if (updates.indexOf(id) === -1) { updates.push(id); } - let value = copy([values[id]])[0]; - let change = { time: time(), source, value }; - if (!(id in changes)) { changes[id] = []; } - changes[id].push(change); + if (DEVTOOLS_HOOK in window) { + let value = copy([values[id]])[0]; + let change = { time: time(), source, value }; + if (!(id in changes)) { changes[id] = []; } + changes[id].push(change); + } +} + +function history(id: number): INodeChange[] { + if (id in changes) { + return changes[id]; + } + return []; } diff --git a/types/core.d.ts b/types/core.d.ts index 0b68d3a9..6e8b58a5 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -14,6 +14,7 @@ export interface IConfig { lookahead?: number; distance?: number; delay?: number; + showText?: boolean; cssRules?: boolean; tokens?: string[]; upload?: (data: string) => void; From abfebf43b6e3a7f48f2d8edc380b8f48d45b32b2 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 21 Aug 2019 08:22:05 -0700 Subject: [PATCH 052/105] Refactoring code to remove viewport directory --- decode/clarity.ts | 4 +- decode/dom.ts | 80 +++++++------ decode/{viewport.ts => interaction.ts} | 6 +- decode/render.ts | 2 +- src/clarity.ts | 3 - src/core/recompute.ts | 5 - src/data/queue.ts | 2 - src/dom/discover.ts | 2 + src/dom/document.ts | 28 +++++ src/dom/encode.ts | 121 +++++++++++--------- src/dom/mutation.ts | 2 + src/interaction/encode.ts | 46 +++++++- src/interaction/index.ts | 6 + src/interaction/mouse.ts | 1 + src/{viewport => interaction}/resize.ts | 3 +- src/{viewport => interaction}/scroll.ts | 2 +- src/{viewport => interaction}/visibility.ts | 2 +- src/viewport/document.ts | 38 ------ src/viewport/encode.ts | 56 --------- src/viewport/index.ts | 15 --- types/dom.d.ts | 5 + types/index.d.ts | 2 +- types/interaction.d.ts | 18 ++- types/viewport.d.ts | 19 --- 24 files changed, 219 insertions(+), 249 deletions(-) rename decode/{viewport.ts => interaction.ts} (75%) delete mode 100644 src/core/recompute.ts create mode 100644 src/dom/document.ts rename src/{viewport => interaction}/resize.ts (90%) rename src/{viewport => interaction}/scroll.ts (96%) rename src/{viewport => interaction}/visibility.ts (90%) delete mode 100644 src/viewport/document.ts delete mode 100644 src/viewport/encode.ts delete mode 100644 src/viewport/index.ts delete mode 100644 types/viewport.d.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 269814fb..7f0e7ab7 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,10 +1,10 @@ import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; import dom from "./dom"; import envelope from "./envelope"; +import interaction from "./interaction"; import metadata from "./metadata"; import metric from "./metric"; import * as r from "./render"; -import viewport from "./viewport"; let pageId: string = null; let payloads: IPayload[] = []; @@ -32,7 +32,7 @@ export function json(data: string): IDecodedPayload { case Event.Scroll: case Event.Document: case Event.Resize: - event = viewport(entry); + event = interaction(entry); break; case Event.Discover: case Event.Mutation: diff --git a/decode/dom.ts b/decode/dom.ts index 28e98f5d..865f169f 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -1,53 +1,59 @@ import { resolve } from "../src/data/token"; import { Event, IDecodedEvent, Token } from "../types/data"; -import { IDecodedNode } from "../types/dom"; +import { IDecodedNode, IDocumentSize } from "../types/dom"; export default function(tokens: Token[]): IDecodedEvent { - let lastType = null; - let node = []; let time = tokens[0] as number; let event = tokens[1] as Event; let decoded: IDecodedEvent = {time, event, data: []}; - let tagIndex = 0; - for (let i = 2; i < tokens.length; i++) { - let token = tokens[i]; - let type = typeof(token); - switch (type) { - case "number": - if (type !== lastType && lastType !== null) { - decoded.data.push(process(node, tagIndex)); - node = []; - tagIndex = 0; - } - node.push(token); - tagIndex++; - break; - case "string": - node.push(token); - break; - case "object": - let subtoken = token[0]; - let subtype = typeof(subtoken); - switch (subtype) { - case "string": - let keys = resolve(token as string); - for (let key of keys) { - node.push(key); + switch (event) { + case Event.Document: + let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; + decoded.data.push(d); + return decoded; + case Event.Discover: + case Event.Mutation: + let lastType = null; + let node = []; + let tagIndex = 0; + for (let i = 2; i < tokens.length; i++) { + let token = tokens[i]; + let type = typeof(token); + switch (type) { + case "number": + if (type !== lastType && lastType !== null) { + decoded.data.push(process(node, tagIndex)); + node = []; + tagIndex = 0; } + node.push(token); + tagIndex++; break; - case "number": - token = tokens.length > subtoken ? tokens[subtoken] : null; + case "string": node.push(token); break; + case "object": + let subtoken = token[0]; + let subtype = typeof(subtoken); + switch (subtype) { + case "string": + let keys = resolve(token as string); + for (let key of keys) { + node.push(key); + } + break; + case "number": + token = tokens.length > subtoken ? tokens[subtoken] : null; + node.push(token); + break; + } } - } - lastType = type; + lastType = type; + } + // Process last node + decoded.data.push(process(node, tagIndex)); + return decoded; } - - // Process last node - decoded.data.push(process(node, tagIndex)); - - return decoded; } function process(node: any[] | number[], tagIndex: number): IDecodedNode { diff --git a/decode/viewport.ts b/decode/interaction.ts similarity index 75% rename from decode/viewport.ts rename to decode/interaction.ts index f314c5a4..d292fdcb 100644 --- a/decode/viewport.ts +++ b/decode/interaction.ts @@ -1,5 +1,5 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -import { IDocumentSize, IResizeViewport, IScrollViewport } from "../types/viewport"; +import { IResizeViewport, IScrollViewport } from "../types/interaction"; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; @@ -9,10 +9,6 @@ export default function(tokens: Token[]): IDecodedEvent { case Event.Resize: let r: IResizeViewport = { width: tokens[2] as number, height: tokens[3] as number }; decoded.data.push(r); - case Event.Document: - let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; - decoded.data.push(d); - break; case Event.Scroll: let t = time; for (let i = 2; i < tokens.length; i = i + 3) { diff --git a/decode/render.ts b/decode/render.ts index b12685f3..b59c82e9 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,6 +1,6 @@ import { IDecodedNode } from "../types/dom"; +import { IResizeViewport, IScrollViewport } from "../types/interaction"; import { IDecodedMetric, IMetricMapValue } from "../types/metric"; -import { IResizeViewport, IScrollViewport } from "../types/viewport"; let nodes = {}; let svgns: string = "http://www.w3.org/2000/svg"; diff --git a/src/clarity.ts b/src/clarity.ts index a96ac783..16ab9f51 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -5,7 +5,6 @@ import * as diagnostic from "@src/diagnostic"; import * as dom from "@src/dom"; import * as interaction from "@src/interaction"; import * as metric from "@src/metric"; -import * as viewport from "@src/viewport"; let status = false; @@ -16,7 +15,6 @@ export function start(configuration: IConfig = {}): void { data.start(); diagnostic.start(); dom.start(); - viewport.start(); interaction.start(); // Mark Clarity session as active @@ -25,7 +23,6 @@ export function start(configuration: IConfig = {}): void { export function end(): void { interaction.end(); - viewport.end(); dom.end(); diagnostic.end(); data.end(); diff --git a/src/core/recompute.ts b/src/core/recompute.ts deleted file mode 100644 index 9c5efd1b..00000000 --- a/src/core/recompute.ts +++ /dev/null @@ -1,5 +0,0 @@ -import * as document from "@src/viewport/document"; - -export default function(): void { - document.compute(); -} diff --git a/src/data/queue.ts b/src/data/queue.ts index ff9a0ccc..d1ab779d 100644 --- a/src/data/queue.ts +++ b/src/data/queue.ts @@ -1,7 +1,6 @@ import { Event, Flush, IEventQueue, Token } from "@clarity-types/data"; import config from "@src/core/config"; import upload from "@src/data/upload"; -import recompute from "../core/recompute"; let events: IEventQueue = { one: [], two: [] }; let timeout: number = null; @@ -40,7 +39,6 @@ export default function(data: Token[], flush: Flush = Flush.Schedule): void { } function dequeue(): void { - recompute(); upload(events); reset(); } diff --git a/src/dom/discover.ts b/src/dom/discover.ts index 1a355d3c..8fb57e66 100644 --- a/src/dom/discover.ts +++ b/src/dom/discover.ts @@ -3,12 +3,14 @@ import { Source } from "@clarity-types/dom"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; +import * as doc from "@src/dom/document"; import encode from "@src/dom/encode"; import processNode from "./node"; export function start(): void { discover().then((data: Token[]) => { + doc.compute(); queue(data); }); } diff --git a/src/dom/document.ts b/src/dom/document.ts new file mode 100644 index 00000000..c9ba7a0c --- /dev/null +++ b/src/dom/document.ts @@ -0,0 +1,28 @@ +import { Event, Token } from "@clarity-types/data"; +import { IDocumentSize } from "@clarity-types/dom"; +import queue from "@src/data/queue"; +import encode from "./encode"; + +export let doc: IDocumentSize; + +export function compute(): void { + let body = document.body; + let d = document.documentElement; + let bodyClientHeight = body ? body.clientHeight : null; + let bodyScrollHeight = body ? body.scrollHeight : null; + let bodyOffsetHeight = body ? body.offsetHeight : null; + let documentClientHeight = d ? d.clientHeight : null; + let documentScrollHeight = d ? d.scrollHeight : null; + let documentOffsetHeight = d ? d.offsetHeight : null; + let documentHeight = Math.max(bodyClientHeight, bodyScrollHeight, bodyOffsetHeight, + documentClientHeight, documentScrollHeight, documentOffsetHeight); + + doc = { + width: body ? body.clientWidth : null, + height: documentHeight + }; + + encode(Event.Document).then((data: Token[]) => { + queue(data); + }); +} diff --git a/src/dom/encode.ts b/src/dom/encode.ts index e9dac9ec..d242fe99 100644 --- a/src/dom/encode.ts +++ b/src/dom/encode.ts @@ -1,75 +1,86 @@ import {Event, Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/dom"; -import { Metric } from "@clarity-types/metric"; +import {Metric} from "@clarity-types/metric"; import config from "@src/core/config"; import * as task from "@src/core/task"; import time from "@src/core/time"; import hash from "@src/data/hash"; import {check} from "@src/data/token"; -import * as metrics from "@src/metric"; - +import * as metric from "@src/metric"; +import {doc} from "./document"; import * as nodes from "./virtualdom"; window["HASH"] = hash; export default async function(type: Event): Promise { - let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; let tokens: Token[] = [time(), type]; - let values = nodes.summarize(); - for (let value of values) { - if (task.longtask(timer)) { await task.idle(timer); } - let metadata = []; - let layouts = []; - let data: INodeData = value.data; - let keys = ["tag", "layout", "attributes", "value"]; - for (let key of keys) { - if (data[key]) { - switch (key) { - case "tag": - metrics.counter(Metric.NodeCount); - tokens.push(value.id); - if (value.parent) { tokens.push(value.parent); } - if (value.next) { tokens.push(value.next); } - metadata.push(data[key]); - break; - case "attributes": - for (let attr in data[key]) { - if (data[key][attr] !== undefined) { - metadata.push(`${attr}=${data[key][attr]}`); - } - } - break; - case "layout": - if (data[key].length > 0) { - let boxes = layout(data[key]); - for (let box of boxes) { - layouts.push(box); - } + let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; + switch (type) { + case Event.Document: + let d = doc; + tokens.push(d.width); + tokens.push(d.height); + metric.measure(Metric.DocumentWidth, d.width); + metric.measure(Metric.DocumentHeight, d.height); + return tokens; + case Event.Discover: + case Event.Mutation: + let values = nodes.summarize(); + for (let value of values) { + if (task.longtask(timer)) { await task.idle(timer); } + let metadata = []; + let layouts = []; + let data: INodeData = value.data; + let keys = ["tag", "layout", "attributes", "value"]; + for (let key of keys) { + if (data[key]) { + switch (key) { + case "tag": + metric.counter(Metric.NodeCount); + tokens.push(value.id); + if (value.parent) { tokens.push(value.parent); } + if (value.next) { tokens.push(value.next); } + metadata.push(data[key]); + break; + case "attributes": + for (let attr in data[key]) { + if (data[key][attr] !== undefined) { + metadata.push(`${attr}=${data[key][attr]}`); + } + } + break; + case "layout": + if (data[key].length > 0) { + let boxes = layout(data[key]); + for (let box of boxes) { + layouts.push(box); + } + } + break; + case "value": + let parent = nodes.getNode(value.parent); + if (parent === null) { console.warn("Unexpected | Node value: " + JSON.stringify(value)); } + let parentTag = nodes.get(parent) ? nodes.get(parent).data.tag : null; + let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; + metadata.push(text(tag, data[key])); + break; } - break; - case "value": - let parent = nodes.getNode(value.parent); - if (parent === null) { console.warn("Unexpected | Node value: " + JSON.stringify(value)); } - let parentTag = nodes.get(parent) ? nodes.get(parent).data.tag : null; - let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; - metadata.push(text(tag, data[key])); - break; + } } - } - } - // Add metadata - metadata = meta(metadata); - for (let token of metadata) { - let index: number = typeof token === "string" ? tokens.indexOf(token) : -1; - tokens.push(index >= 0 && token.length > index.toString().length ? [index] : token); - } - // Add layout boxes - for (let entry of layouts) { - tokens.push(entry); + // Add metadata + metadata = meta(metadata); + for (let token of metadata) { + let index: number = typeof token === "string" ? tokens.indexOf(token) : -1; + tokens.push(index >= 0 && token.length > index.toString().length ? [index] : token); + } + // Add layout boxes + for (let entry of layouts) { + tokens.push(entry); + } + } + return tokens; } - } - return tokens; } function meta(metadata: string[]): string[] | string[][] { diff --git a/src/dom/mutation.ts b/src/dom/mutation.ts index dc6fcf19..0c4ad027 100644 --- a/src/dom/mutation.ts +++ b/src/dom/mutation.ts @@ -3,6 +3,7 @@ import { Source } from "@clarity-types/dom"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; +import * as doc from "@src/dom/document"; import encode from "@src/dom/encode"; import processNode from "./node"; @@ -23,6 +24,7 @@ export function end(): void { function handle(mutations: MutationRecord[]): void { process(mutations).then((data: Token[]) => { + doc.compute(); queue(data); }); } diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index a18303be..d33500f2 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -1,22 +1,27 @@ import {Event, Token} from "@clarity-types/data"; import { Mouse } from "@clarity-types/interaction"; +import {Metric} from "@clarity-types/metric"; +import time from "@src/core/time"; +import * as metric from "@src/metric"; import * as mouse from "./mouse"; +import * as resize from "./resize"; +import * as scroll from "./scroll"; +import * as visibility from "./visibility"; export default function(type: Event): Token[] { - let tokens: Token[] = []; - + let tokens: Token[] = [time(), type]; + let timestamp: number = null; switch (type) { case Event.Mouse: let m = mouse.summarize(); - let timestamp: number = null; + timestamp = null; let mouseType: Mouse = null; for (let i = 0; i < m.length; i++) { let entry = m[i]; if (i === 0) { timestamp = entry.time; - tokens.push(timestamp); - tokens.push(type); + tokens[0] = timestamp; } if (mouseType !== entry.type) { tokens.push(entry.type); } @@ -31,6 +36,37 @@ export default function(type: Event): Token[] { } mouse.reset(); break; + case Event.Resize: + let r = resize.data; + tokens.push(r.width); + tokens.push(r.height); + metric.measure(Metric.ViewportWidth, r.width); + metric.measure(Metric.ViewportHeight, r.height); + resize.reset(); + break; + case Event.Scroll: + let s = scroll.summarize(); + timestamp = null; + for (let i = 0; i < s.length; i++) { + let entry = s[i]; + + if (i === 0) { + timestamp = entry.time; + tokens[0] = timestamp; + } + + tokens.push(entry.time - timestamp); + tokens.push(entry.x); + tokens.push(entry.y); + timestamp = entry.time; + } + scroll.reset(); + break; + case Event.Visibility: + let v = visibility.data; + tokens.push(v.visible); + visibility.reset(); + break; } return tokens; diff --git a/src/interaction/index.ts b/src/interaction/index.ts index 1dc5d8ad..c670ebf0 100644 --- a/src/interaction/index.ts +++ b/src/interaction/index.ts @@ -1,7 +1,13 @@ import * as mouse from "@src/interaction/mouse"; +import * as resize from "@src/interaction/resize"; +import * as scroll from "@src/interaction/scroll"; +import * as visibility from "@src/interaction/visibility"; export function start(): void { mouse.start(); + resize.start(); + visibility.start(); + scroll.start(); } export function end(): void { diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index a1a870ff..fb6b2c33 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -15,6 +15,7 @@ export function start(): void { bind(document, "mouseup", handler.bind(this, Mouse.Up)); bind(document, "mousemove", handler.bind(this, Mouse.Move)); bind(document, "mousewheel", handler.bind(this, Mouse.Wheel)); + bind(document, "dblclick", handler.bind(this, Mouse.DoubleClick)); bind(document, "click", handler.bind(this, Mouse.Click)); } diff --git a/src/viewport/resize.ts b/src/interaction/resize.ts similarity index 90% rename from src/viewport/resize.ts rename to src/interaction/resize.ts index ddcd0ba8..94d1f981 100644 --- a/src/viewport/resize.ts +++ b/src/interaction/resize.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; -import { IResizeViewport } from "@clarity-types/viewport"; +import { IResizeViewport } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; - import queue from "@src/data/queue"; import encode from "./encode"; diff --git a/src/viewport/scroll.ts b/src/interaction/scroll.ts similarity index 96% rename from src/viewport/scroll.ts rename to src/interaction/scroll.ts index 1a241ee2..28e4f708 100644 --- a/src/viewport/scroll.ts +++ b/src/interaction/scroll.ts @@ -1,5 +1,5 @@ import { Event } from "@clarity-types/data"; -import { IScrollViewport } from "@clarity-types/viewport"; +import { IScrollViewport } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; diff --git a/src/viewport/visibility.ts b/src/interaction/visibility.ts similarity index 90% rename from src/viewport/visibility.ts rename to src/interaction/visibility.ts index c044a5ac..3df4dec8 100644 --- a/src/viewport/visibility.ts +++ b/src/interaction/visibility.ts @@ -1,5 +1,5 @@ import { Event } from "@clarity-types/data"; -import { IPageVisibility } from "@clarity-types/viewport"; +import { IPageVisibility } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; import queue from "@src/data/queue"; import encode from "./encode"; diff --git a/src/viewport/document.ts b/src/viewport/document.ts deleted file mode 100644 index a14038d4..00000000 --- a/src/viewport/document.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Event, Flush } from "@clarity-types/data"; -import { IDocumentSize } from "@clarity-types/viewport"; -import queue from "@src/data/queue"; -import encode from "./encode"; - -export let data: IDocumentSize; - -export function start(): void { - recompute(); -} - -function recompute(): void { - let body = document.body; - let doc = document.documentElement; - let bodyClientHeight = body ? body.clientHeight : null; - let bodyScrollHeight = body ? body.scrollHeight : null; - let bodyOffsetHeight = body ? body.offsetHeight : null; - let documentClientHeight = doc ? doc.clientHeight : null; - let documentScrollHeight = doc ? doc.scrollHeight : null; - let documentOffsetHeight = doc ? doc.offsetHeight : null; - let documentHeight = Math.max(bodyClientHeight, bodyScrollHeight, bodyOffsetHeight, - documentClientHeight, documentScrollHeight, documentOffsetHeight); - - data = { - width: body ? body.clientWidth : null, - height: documentHeight - }; - - queue(encode(Event.Document), Flush.None); -} - -export function compute(): void { - recompute(); -} - -export function reset(): void { - data = null; -} diff --git a/src/viewport/encode.ts b/src/viewport/encode.ts deleted file mode 100644 index 52b8008f..00000000 --- a/src/viewport/encode.ts +++ /dev/null @@ -1,56 +0,0 @@ -import {Event, Token} from "@clarity-types/data"; -import {Metric} from "@clarity-types/metric"; -import time from "@src/core/time"; -import * as metric from "@src/metric"; -import * as document from "./document"; -import * as resize from "./resize"; -import * as scroll from "./scroll"; -import * as visibility from "./visibility"; - -export default function(type: Event): Token[] { - let tokens: Token[] = [time(), type]; - - switch (type) { - case Event.Resize: - let r = resize.data; - tokens.push(r.width); - tokens.push(r.height); - metric.measure(Metric.ViewportWidth, r.width); - metric.measure(Metric.ViewportHeight, r.height); - resize.reset(); - break; - case Event.Document: - let d = document.data; - tokens.push(d.width); - tokens.push(d.height); - metric.measure(Metric.DocumentWidth, d.width); - metric.measure(Metric.DocumentHeight, d.height); - document.reset(); - break; - case Event.Scroll: - let s = scroll.summarize(); - let timestamp: number = null; - for (let i = 0; i < s.length; i++) { - let entry = s[i]; - - if (i === 0) { - timestamp = entry.time; - tokens[0] = timestamp; - } - - tokens.push(entry.time - timestamp); - tokens.push(entry.x); - tokens.push(entry.y); - timestamp = entry.time; - } - scroll.reset(); - break; - case Event.Visibility: - let v = visibility.data; - tokens.push(v.visible); - visibility.reset(); - break; - } - - return tokens; -} diff --git a/src/viewport/index.ts b/src/viewport/index.ts deleted file mode 100644 index d96aee41..00000000 --- a/src/viewport/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -import * as document from "@src/viewport/document"; -import * as resize from "@src/viewport/resize"; -import * as scroll from "@src/viewport/scroll"; -import * as visibility from "@src/viewport/visibility"; - -export function start(): void { - document.start(); - resize.start(); - visibility.start(); - scroll.start(); -} - -export function end(): void { - // End calls -} diff --git a/types/dom.d.ts b/types/dom.d.ts index c1c8e01a..2c1bfc65 100644 --- a/types/dom.d.ts +++ b/types/dom.d.ts @@ -40,3 +40,8 @@ export interface IDecodedNode { layout?: number[][]; value?: string; } + +export interface IDocumentSize { + width: number; + height: number; +} diff --git a/types/index.d.ts b/types/index.d.ts index 4742ca7f..eb92728a 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -10,9 +10,9 @@ interface IClarityJs { declare const clarity: IClarityJs; export * from "./data"; +export * from "./diagnostic"; export * from "./dom"; export * from "./interaction"; export * from "./metric"; -export * from "./viewport"; export { clarity }; diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 61fef632..1c4202f4 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -3,10 +3,11 @@ export const enum Mouse { Up = "u", Move = "m", Wheel = "w", + DoubleClick = "b", Click = "c" } -interface IMouseInteraction { +export interface IMouseInteraction { type: Mouse; time: number; x: number; @@ -14,3 +15,18 @@ interface IMouseInteraction { target: number; buttons: number; } + +export interface IResizeViewport { + width: number; + height: number; +} + +export interface IScrollViewport { + time: number; + x: number; + y: number; +} + +export interface IPageVisibility { + visible: string; +} diff --git a/types/viewport.d.ts b/types/viewport.d.ts deleted file mode 100644 index fd5f7d1f..00000000 --- a/types/viewport.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -export interface IResizeViewport { - width: number; - height: number; -} - -export interface IScrollViewport { - time: number; - x: number; - y: number; -} - -export interface IDocumentSize { - width: number; - height: number; -} - -export interface IPageVisibility { - visible: string; -} From a23af3c426bb44829c73ac2c24c8ebd48e0f3c10 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 22 Aug 2019 07:36:35 -0700 Subject: [PATCH 053/105] Simplifying events and improving text masking --- decode/dom.ts | 15 +++++++++++- decode/interaction.ts | 27 ++++++++++++++-------- decode/render.ts | 14 ++++++++---- src/interaction/encode.ts | 26 ++++++++++++++++----- src/interaction/mouse.ts | 12 +++++----- src/interaction/resize.ts | 4 ++-- src/interaction/scroll.ts | 48 +++++++++++++++++++++++++++------------ types/interaction.d.ts | 30 ++++++++++++++---------- 8 files changed, 122 insertions(+), 54 deletions(-) diff --git a/decode/dom.ts b/decode/dom.ts index 865f169f..1e56df7f 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -89,7 +89,20 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let textCount = parseInt(parts[0], 36); let wordCount = parseInt(parts[1], 36); if (isFinite(textCount) && isFinite(wordCount)) { - value = wordCount > 0 && textCount === 0 ? " " : Array(Math.floor((textCount + 1) / 2)).join("* "); + if (wordCount > 0 && textCount === 0) { + value = " "; + } else if (wordCount === 0 && textCount > 0) { + value = Array(textCount).join("*"); + } else if (wordCount > 0 && textCount > 0) { + value = ""; + let avg = Math.floor(textCount / wordCount); + while (value.length < textCount + wordCount) { + let gap = Math.min(avg, textCount + wordCount - value.length); + value += Array(gap).join("*") + " "; + } + } else { + value = Array(Math.floor((textCount + 1) / 2)).join("* "); + } } } } diff --git a/decode/interaction.ts b/decode/interaction.ts index d292fdcb..b55eff48 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -1,5 +1,5 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -import { IResizeViewport, IScrollViewport } from "../types/interaction"; +import { IResize, IScroll, Scroll } from "../types/interaction"; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; @@ -7,16 +7,25 @@ export default function(tokens: Token[]): IDecodedEvent { let decoded: IDecodedEvent = {time, event, data: []}; switch (event) { case Event.Resize: - let r: IResizeViewport = { width: tokens[2] as number, height: tokens[3] as number }; + let r: IResize = { width: tokens[2] as number, height: tokens[3] as number }; decoded.data.push(r); case Event.Scroll: - let t = time; - for (let i = 2; i < tokens.length; i = i + 3) { - t += tokens[i] as number; - let s: IScrollViewport = { time: t, x: tokens[i + 1] as number, y: tokens[i + 2] as number }; - decoded.data.push(s); - } - break; + let i = 2; + let scrollType = null; + let target = null; + let t = 0; + while (i < tokens.length) { + if (typeof(tokens[i]) === "string") { + scrollType = tokens[i++] as Scroll; + target = tokens[i++] as number; + continue; + } + t += tokens[i++] as number; + let v = tokens[i++] as number; + let s: IScroll = { type: scrollType, target, time: t, value: v }; + decoded.data.push(s); + } + break; } return decoded; } diff --git a/decode/render.ts b/decode/render.ts index b59c82e9..5f8bd70d 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ import { IDecodedNode } from "../types/dom"; -import { IResizeViewport, IScrollViewport } from "../types/interaction"; +import { IResize, IScroll, Scroll } from "../types/interaction"; import { IDecodedMetric, IMetricMapValue } from "../types/metric"; let nodes = {}; @@ -172,13 +172,15 @@ function setAttributes(node: HTMLElement, attributes: object): void { } } -export function scroll(data: IScrollViewport[], placeholder: HTMLIFrameElement): void { +export function scroll(data: IScroll[], iframe: HTMLIFrameElement): void { for (let d of data) { - placeholder.contentWindow.scrollTo(d.x, d.y); + let target = getNode(d.target); + if (target && d.type === Scroll.X) { target.scrollTo(d.value, target.scrollTop); } + if (target && d.type === Scroll.Y) { target.scrollTo(target.scrollLeft, d.value); } } } -export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): void { +export function resize(data: IResize, placeholder: HTMLIFrameElement): void { placeholder.removeAttribute("style"); let margin = 10; let px = "px"; @@ -198,3 +200,7 @@ export function resize(data: IResizeViewport, placeholder: HTMLIFrameElement): v placeholder.style.margin = margin + px; placeholder.style.overflow = "hidden"; } + +function getNode(id: number): HTMLElement { + return id in nodes ? nodes[id] : null; +} diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index d33500f2..56e46e63 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -1,5 +1,5 @@ import {Event, Token} from "@clarity-types/data"; -import { Mouse } from "@clarity-types/interaction"; +import {Mouse, Scroll} from "@clarity-types/interaction"; import {Metric} from "@clarity-types/metric"; import time from "@src/core/time"; import * as metric from "@src/metric"; @@ -16,6 +16,7 @@ export default function(type: Event): Token[] { let m = mouse.summarize(); timestamp = null; let mouseType: Mouse = null; + let mouseTarget: number = null; for (let i = 0; i < m.length; i++) { let entry = m[i]; @@ -24,15 +25,19 @@ export default function(type: Event): Token[] { tokens[0] = timestamp; } - if (mouseType !== entry.type) { tokens.push(entry.type); } + if (mouseType !== entry.type || mouseTarget !== entry.target) { + tokens.push(entry.type); + tokens.push(entry.target); + mouseType = entry.type; + mouseTarget = entry.target; + } + tokens.push(entry.time - timestamp); tokens.push(entry.x); tokens.push(entry.y); - tokens.push(entry.target); tokens.push(entry.buttons); timestamp = entry.time; - mouseType = entry.type; } mouse.reset(); break; @@ -46,6 +51,8 @@ export default function(type: Event): Token[] { break; case Event.Scroll: let s = scroll.summarize(); + let scrollType: Scroll = null; + let scrollTarget: number = null; timestamp = null; for (let i = 0; i < s.length; i++) { let entry = s[i]; @@ -55,9 +62,16 @@ export default function(type: Event): Token[] { tokens[0] = timestamp; } + if (scrollType !== entry.type || scrollTarget !== entry.target) { + tokens.push(entry.type); + tokens.push(entry.target); + scrollType = entry.type; + scrollTarget = entry.target; + } + tokens.push(entry.time - timestamp); - tokens.push(entry.x); - tokens.push(entry.y); + tokens.push(entry.value); + timestamp = entry.time; } scroll.reset(); diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index fb6b2c33..84cc05a5 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -1,5 +1,5 @@ import { Event } from "@clarity-types/data"; -import { IMouseInteraction, Mouse } from "@clarity-types/interaction"; +import { IMouse, Mouse } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; @@ -7,7 +7,7 @@ import queue from "@src/data/queue"; import { getId } from "@src/dom/virtualdom"; import encode from "./encode"; -let data: IMouseInteraction[] = []; +let data: IMouse[] = []; let timeout: number = null; export function start(): void { @@ -23,10 +23,10 @@ function handler(type: Mouse, evt: MouseEvent): void { let de = document.documentElement; data.push({ type, + target: evt.target ? getId(evt.target as Node) : null, time: time(), x: "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null), y: "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null), - target: evt.target ? getId(evt.target as Node) : null, buttons: evt.buttons }); if (timeout) { clearTimeout(timeout); } @@ -41,8 +41,8 @@ export function reset(): void { data = []; } -export function summarize(): IMouseInteraction[] { - let summary: IMouseInteraction[] = []; +export function summarize(): IMouse[] { + let summary: IMouse[] = []; let index = 0; let last = null; for (let entry of data) { @@ -58,7 +58,7 @@ export function summarize(): IMouseInteraction[] { return summary; } -function checkDistance(last: IMouseInteraction, current: IMouseInteraction): boolean { +function checkDistance(last: IMouse, current: IMouse): boolean { let dx = last.x - current.x; let dy = last.y - current.y; return (dx * dx + dy * dy > config.distance * config.distance); diff --git a/src/interaction/resize.ts b/src/interaction/resize.ts index 94d1f981..fe8304b1 100644 --- a/src/interaction/resize.ts +++ b/src/interaction/resize.ts @@ -1,10 +1,10 @@ import { Event } from "@clarity-types/data"; -import { IResizeViewport } from "@clarity-types/interaction"; +import { IResize } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; import queue from "@src/data/queue"; import encode from "./encode"; -export let data: IResizeViewport; +export let data: IResize; export function start(): void { bind(window, "resize", recompute); diff --git a/src/interaction/scroll.ts b/src/interaction/scroll.ts index 28e4f708..b2f283f3 100644 --- a/src/interaction/scroll.ts +++ b/src/interaction/scroll.ts @@ -1,23 +1,40 @@ import { Event } from "@clarity-types/data"; -import { IScrollViewport } from "@clarity-types/interaction"; +import { IScroll, Scroll } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; import queue from "@src/data/queue"; +import { getId } from "@src/dom/virtualdom"; import encode from "./encode"; -let data: IScrollViewport[] = []; +let lastX = {}; +let lastY = {}; +let dataX: IScroll[] = []; +let dataY: IScroll[] = []; let timeout: number = null; export function start(): void { - bind(window, "scroll", recompute); + bind(window, "scroll", recompute, true); recompute(); } -function recompute(): void { - let x = "pageXOffset" in window ? window.pageXOffset : document.documentElement.scrollLeft; - let y = "pageYOffset" in window ? window.pageYOffset : document.documentElement.scrollTop; - data.push({ time: time(), x, y }); +function recompute(event: UIEvent = null): void { + let t = time(); + let eventTarget = event ? (event.target === document ? document.documentElement : event.target) : document.documentElement; + let target = getId(eventTarget as Node); + let x = (eventTarget as HTMLElement).scrollLeft; + let y = (eventTarget as HTMLElement).scrollTop; + + if (x !== lastX[target]) { + dataX.push({ target, type: Scroll.X, time: t, value: x }); + lastX[target] = x; + } + + if (y !== lastY[target]) { + dataY.push({ target, type: Scroll.Y, time: t, value: y }); + lastY[target] = y; + } + if (timeout) { clearTimeout(timeout); } timeout = window.setTimeout(schedule, config.lookahead); } @@ -27,12 +44,16 @@ function schedule(): void { } export function reset(): void { - data = []; + dataX = []; + dataY = []; + lastX = {}; + lastY = {}; } -export function summarize(): IScrollViewport[] { - let summary: IScrollViewport[] = []; +export function summarize(): IScroll[] { + let summary: IScroll[] = []; let last = null; + let data = dataX.concat(dataY); for (let i = 0; i < data.length; i++) { let entry = data[i]; let isFirst = i === 0; @@ -45,8 +66,7 @@ export function summarize(): IScrollViewport[] { return summary; } -function checkDistance(last: IScrollViewport, current: IScrollViewport): boolean { - let dx = last.x - current.x; - let dy = last.y - current.y; - return (dx * dx + dy * dy > config.distance * config.distance); +function checkDistance(last: IScroll, current: IScroll): boolean { + let d = last.value - current.value; + return (d > config.distance); } diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 1c4202f4..d6219c24 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -1,30 +1,36 @@ export const enum Mouse { - Down = "d", - Up = "u", - Move = "m", - Wheel = "w", - DoubleClick = "b", - Click = "c" + Down = "D", + Up = "U", + Move = "M", + Wheel = "W", + DoubleClick = "B", + Click = "C" } -export interface IMouseInteraction { +export interface IMouse { type: Mouse; + target: number; time: number; x: number; y: number; - target: number; buttons: number; } -export interface IResizeViewport { +export interface IResize { width: number; height: number; } -export interface IScrollViewport { +export const enum Scroll { + X = "X", + Y = "Y" +} + +export interface IScroll { + type: Scroll; + target: number; time: number; - x: number; - y: number; + value: number; } export interface IPageVisibility { From 2ffe9c1fa09bf9ed092483015d502213ac20c6a8 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 22 Aug 2019 19:33:05 -0700 Subject: [PATCH 054/105] Removing debug statements --- decode/clarity.ts | 1 - decode/dom.ts | 1 - decode/render.ts | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 7f0e7ab7..3ae19020 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -58,7 +58,6 @@ export function html(data: string): string { export function render(data: string, iframe: HTMLIFrameElement, header?: HTMLElement): void { let decoded = json(data); - console.log("Decoded: " + JSON.stringify(decoded)); // Render metrics r.metrics(decoded.metrics, header); diff --git a/decode/dom.ts b/decode/dom.ts index 1e56df7f..ddac6a4b 100644 --- a/decode/dom.ts +++ b/decode/dom.ts @@ -67,7 +67,6 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let layouts = []; let attributes = {}; let value = null; - console.log("Node: " + JSON.stringify(node) + " | Tag: " + tagIndex); for (let i = tagIndex + 1; i < node.length; i++) { let token = node[i] as string; let keyIndex = token.indexOf("="); diff --git a/decode/render.ts b/decode/render.ts index 5f8bd70d..6ef93447 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -83,7 +83,7 @@ export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { if (doc.head) { doc.head.parentNode.removeChild(doc.head); } if (doc.body) { doc.body.parentNode.removeChild(doc.body); } } - setAttributes(docElement as HTMLElement, node.attributes); + setAttributes(doc.documentElement as HTMLElement, node.attributes); nodes[node.id] = doc.documentElement; break; case "HEAD": From 88d5decb637e589fce12b7da5fac4c373c55510f Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 23 Aug 2019 18:15:19 -0700 Subject: [PATCH 055/105] Separate logic to decode JSON and render JSON --- decode/clarity.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 3ae19020..ec052d90 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -50,15 +50,13 @@ export function json(data: string): IDecodedPayload { return decoded; } -export function html(data: string): string { +export function html(decoded: IDecodedPayload): string { let iframe = document.createElement("iframe"); - render(data, iframe); + render(decoded, iframe); return iframe.contentDocument.documentElement.outerHTML; } -export function render(data: string, iframe: HTMLIFrameElement, header?: HTMLElement): void { - let decoded = json(data); - +export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, header?: HTMLElement): void { // Render metrics r.metrics(decoded.metrics, header); From 700dbb712fa29b4978ece2cfdb1fc92246be699c Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sat, 24 Aug 2019 08:59:30 -0700 Subject: [PATCH 056/105] Refactoring dom directory to layout --- decode/clarity.ts | 4 ++-- decode/{dom.ts => layout.ts} | 6 +++--- decode/render.ts | 2 +- src/clarity.ts | 2 +- src/diagnostic/image.ts | 2 +- src/dom/index.ts | 14 -------------- src/interaction/mouse.ts | 2 +- src/interaction/scroll.ts | 2 +- src/{dom => layout}/discover.ts | 6 +++--- src/{dom => layout}/document.ts | 2 +- src/{dom/virtualdom.ts => layout/dom.ts} | 2 +- src/{dom => layout}/encode.ts | 4 ++-- src/layout/index.ts | 14 ++++++++++++++ src/{dom => layout}/mutation.ts | 6 +++--- src/{dom => layout}/node.ts | 20 ++++++++++---------- types/index.d.ts | 2 +- types/{dom.d.ts => layout.d.ts} | 0 17 files changed, 45 insertions(+), 45 deletions(-) rename decode/{dom.ts => layout.ts} (96%) delete mode 100644 src/dom/index.ts rename src/{dom => layout}/discover.ts (85%) rename src/{dom => layout}/document.ts (94%) rename src/{dom/virtualdom.ts => layout/dom.ts} (99%) rename src/{dom => layout}/encode.ts (98%) create mode 100644 src/layout/index.ts rename src/{dom => layout}/mutation.ts (94%) rename src/{dom => layout}/node.ts (89%) rename types/{dom.d.ts => layout.d.ts} (100%) diff --git a/decode/clarity.ts b/decode/clarity.ts index ec052d90..05ef2631 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,7 +1,7 @@ import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; -import dom from "./dom"; import envelope from "./envelope"; import interaction from "./interaction"; +import layout from "./layout"; import metadata from "./metadata"; import metric from "./metric"; import * as r from "./render"; @@ -36,7 +36,7 @@ export function json(data: string): IDecodedPayload { break; case Event.Discover: case Event.Mutation: - event = dom(entry); + event = layout(entry); break; case Event.Metadata: event = metadata(entry); diff --git a/decode/dom.ts b/decode/layout.ts similarity index 96% rename from decode/dom.ts rename to decode/layout.ts index ddac6a4b..10758ddd 100644 --- a/decode/dom.ts +++ b/decode/layout.ts @@ -1,6 +1,6 @@ import { resolve } from "../src/data/token"; import { Event, IDecodedEvent, Token } from "../types/data"; -import { IDecodedNode, IDocumentSize } from "../types/dom"; +import { IDecodedNode, IDocumentSize } from "../types/layout"; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; @@ -97,10 +97,10 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let avg = Math.floor(textCount / wordCount); while (value.length < textCount + wordCount) { let gap = Math.min(avg, textCount + wordCount - value.length); - value += Array(gap).join("*") + " "; + value += Array(gap).join("x") + " "; } } else { - value = Array(Math.floor((textCount + 1) / 2)).join("* "); + value = Array(Math.floor((textCount + 1) / 2)).join("x "); } } } diff --git a/decode/render.ts b/decode/render.ts index 6ef93447..adb87e78 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ -import { IDecodedNode } from "../types/dom"; import { IResize, IScroll, Scroll } from "../types/interaction"; +import { IDecodedNode } from "../types/layout"; import { IDecodedMetric, IMetricMapValue } from "../types/metric"; let nodes = {}; diff --git a/src/clarity.ts b/src/clarity.ts index 16ab9f51..2da55edc 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -2,8 +2,8 @@ import { IConfig } from "@clarity-types/core"; import * as core from "@src/core"; import * as data from "@src/data"; import * as diagnostic from "@src/diagnostic"; -import * as dom from "@src/dom"; import * as interaction from "@src/interaction"; +import * as dom from "@src/layout"; import * as metric from "@src/metric"; let status = false; diff --git a/src/diagnostic/image.ts b/src/diagnostic/image.ts index 2804600e..8a6ccdec 100644 --- a/src/diagnostic/image.ts +++ b/src/diagnostic/image.ts @@ -2,7 +2,7 @@ import { Event } from "@clarity-types/data"; import { IImageError } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; import queue from "@src/data/queue"; -import { getId } from "@src/dom/virtualdom"; +import { getId } from "@src/layout/dom"; import encode from "./encode"; export let data: IImageError[] = []; diff --git a/src/dom/index.ts b/src/dom/index.ts deleted file mode 100644 index 6a4155dc..00000000 --- a/src/dom/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -import * as discover from "@src/dom/discover"; -import * as mutation from "@src/dom/mutation"; -import * as virtualdom from "@src/dom/virtualdom"; - -export function start(): void { - virtualdom.reset(); - mutation.start(); - discover.start(); -} - -export function end(): void { - virtualdom.reset(); - mutation.end(); -} diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index 84cc05a5..df859391 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -4,7 +4,7 @@ import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; import queue from "@src/data/queue"; -import { getId } from "@src/dom/virtualdom"; +import { getId } from "@src/layout/dom"; import encode from "./encode"; let data: IMouse[] = []; diff --git a/src/interaction/scroll.ts b/src/interaction/scroll.ts index b2f283f3..cceeb9b2 100644 --- a/src/interaction/scroll.ts +++ b/src/interaction/scroll.ts @@ -4,7 +4,7 @@ import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; import queue from "@src/data/queue"; -import { getId } from "@src/dom/virtualdom"; +import { getId } from "@src/layout/dom"; import encode from "./encode"; let lastX = {}; diff --git a/src/dom/discover.ts b/src/layout/discover.ts similarity index 85% rename from src/dom/discover.ts rename to src/layout/discover.ts index 8fb57e66..72c09c02 100644 --- a/src/dom/discover.ts +++ b/src/layout/discover.ts @@ -1,10 +1,10 @@ import { Event, Token } from "@clarity-types/data"; -import { Source } from "@clarity-types/dom"; +import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; -import * as doc from "@src/dom/document"; -import encode from "@src/dom/encode"; +import * as doc from "@src/layout/document"; +import encode from "@src/layout/encode"; import processNode from "./node"; diff --git a/src/dom/document.ts b/src/layout/document.ts similarity index 94% rename from src/dom/document.ts rename to src/layout/document.ts index c9ba7a0c..9572189d 100644 --- a/src/dom/document.ts +++ b/src/layout/document.ts @@ -1,5 +1,5 @@ import { Event, Token } from "@clarity-types/data"; -import { IDocumentSize } from "@clarity-types/dom"; +import { IDocumentSize } from "@clarity-types/layout"; import queue from "@src/data/queue"; import encode from "./encode"; diff --git a/src/dom/virtualdom.ts b/src/layout/dom.ts similarity index 99% rename from src/dom/virtualdom.ts rename to src/layout/dom.ts index 75b05625..8ea3f5cc 100644 --- a/src/dom/virtualdom.ts +++ b/src/layout/dom.ts @@ -1,4 +1,4 @@ -import { INodeChange, INodeData, INodeValue, Source } from "@clarity-types/dom"; +import { INodeChange, INodeData, INodeValue, Source } from "@clarity-types/layout"; import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; diff --git a/src/dom/encode.ts b/src/layout/encode.ts similarity index 98% rename from src/dom/encode.ts rename to src/layout/encode.ts index d242fe99..4c0e29a9 100644 --- a/src/dom/encode.ts +++ b/src/layout/encode.ts @@ -1,5 +1,5 @@ import {Event, Token} from "@clarity-types/data"; -import {INodeData} from "@clarity-types/dom"; +import {INodeData} from "@clarity-types/layout"; import {Metric} from "@clarity-types/metric"; import config from "@src/core/config"; import * as task from "@src/core/task"; @@ -8,7 +8,7 @@ import hash from "@src/data/hash"; import {check} from "@src/data/token"; import * as metric from "@src/metric"; import {doc} from "./document"; -import * as nodes from "./virtualdom"; +import * as nodes from "./dom"; window["HASH"] = hash; diff --git a/src/layout/index.ts b/src/layout/index.ts new file mode 100644 index 00000000..9cde62fb --- /dev/null +++ b/src/layout/index.ts @@ -0,0 +1,14 @@ +import * as discover from "@src/layout/discover"; +import * as dom from "@src/layout/dom"; +import * as mutation from "@src/layout/mutation"; + +export function start(): void { + dom.reset(); + mutation.start(); + discover.start(); +} + +export function end(): void { + dom.reset(); + mutation.end(); +} diff --git a/src/dom/mutation.ts b/src/layout/mutation.ts similarity index 94% rename from src/dom/mutation.ts rename to src/layout/mutation.ts index 0c4ad027..36b24d54 100644 --- a/src/dom/mutation.ts +++ b/src/layout/mutation.ts @@ -1,10 +1,10 @@ import { Event, Token } from "@clarity-types/data"; -import { Source } from "@clarity-types/dom"; +import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; -import * as doc from "@src/dom/document"; -import encode from "@src/dom/encode"; +import * as doc from "@src/layout/document"; +import encode from "@src/layout/encode"; import processNode from "./node"; let observer: MutationObserver; diff --git a/src/dom/node.ts b/src/layout/node.ts similarity index 89% rename from src/dom/node.ts rename to src/layout/node.ts index 0def7ab4..b687fd63 100644 --- a/src/dom/node.ts +++ b/src/layout/node.ts @@ -1,20 +1,20 @@ -import { Source } from "@clarity-types/dom"; +import { Source } from "@clarity-types/layout"; import config from "@src/core/config"; -import * as virtualdom from "./virtualdom"; +import * as dom from "./dom"; let ignoreAttributes = ["title", "alt", "onload", "onfocus"]; export default function(node: Node, source: Source): void { // Do not track this change if we are attempting to remove a node before discovering it - if (source === Source.ChildListRemove && virtualdom.has(node) === false) { return; } + if (source === Source.ChildListRemove && dom.has(node) === false) { return; } - let call = virtualdom.has(node) ? "update" : "add"; + let call = dom.has(node) ? "update" : "add"; switch (node.nodeType) { case Node.DOCUMENT_TYPE_NODE: let doctype = node as DocumentType; let docAttributes = { name: doctype.name, publicId: doctype.publicId, systemId: doctype.systemId }; let docData = { tag: "*D", attributes: docAttributes }; - virtualdom[call](node, docData, source); + dom[call](node, docData, source); break; case Node.TEXT_NODE: // Account for this text node only if we are tracking the parent node @@ -23,10 +23,10 @@ export default function(node: Node, source: Source): void { // The only exception is when we receive a mutation to remove the text node, in that case // parent will be null, but we can still process the node by checking it's an update call. let parent = node.parentElement; - if (call === "update" || (parent && virtualdom.has(parent) && parent.tagName !== "STYLE")) { + if (call === "update" || (parent && dom.has(parent) && parent.tagName !== "STYLE")) { let textData = { tag: "*T", value: node.nodeValue }; textData["layout"] = getTextLayout(node); - virtualdom[call](node, textData, source); + dom[call](node, textData, source); } break; case Node.ELEMENT_NODE: @@ -41,17 +41,17 @@ export default function(node: Node, source: Source): void { let head = { tag, attributes: getAttributes(element.attributes) }; // Capture base href as part of discovering DOM if (call === "add") { head.attributes["*B"] = location.protocol + "//" + location.hostname; } - virtualdom[call](node, head, source); + dom[call](node, head, source); break; case "STYLE": let attributes = getAttributes(element.attributes); let styleData = { tag, attributes, value: getStyleValue(element as HTMLStyleElement) }; - virtualdom[call](node, styleData, source); + dom[call](node, styleData, source); break; default: let data = { tag, attributes: getAttributes(element.attributes) }; data["layout"] = getLayout(element); - virtualdom[call](node, data, source); + dom[call](node, data, source); break; } break; diff --git a/types/index.d.ts b/types/index.d.ts index eb92728a..d910d825 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -11,7 +11,7 @@ declare const clarity: IClarityJs; export * from "./data"; export * from "./diagnostic"; -export * from "./dom"; +export * from "./layout"; export * from "./interaction"; export * from "./metric"; diff --git a/types/dom.d.ts b/types/layout.d.ts similarity index 100% rename from types/dom.d.ts rename to types/layout.d.ts From dd0dc9d16540c82fc33237fb9cc4cbff57376c84 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 26 Aug 2019 07:48:31 -0700 Subject: [PATCH 057/105] Adding basic support for box model --- decode/clarity.ts | 4 ++ decode/layout.ts | 22 ++++---- decode/metric.ts | 19 ++++--- decode/render.ts | 13 ++++- src/core/config.ts | 6 +-- src/core/task.ts | 9 +++- src/data/encode.ts | 2 +- src/data/upload.ts | 19 +++---- src/diagnostic/encode.ts | 4 +- src/layout/boxmodel.ts | 110 +++++++++++++++++++++++++++++++++++++++ src/layout/discover.ts | 4 +- src/layout/dom.ts | 43 +++++++++------ src/layout/encode.ts | 18 +++++-- src/layout/mutation.ts | 2 + src/layout/node.ts | 47 ----------------- types/data.d.ts | 1 + types/layout.d.ts | 10 +++- types/metric.d.ts | 21 ++++---- 18 files changed, 237 insertions(+), 117 deletions(-) create mode 100644 src/layout/boxmodel.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 05ef2631..b64e3de0 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -36,6 +36,7 @@ export function json(data: string): IDecodedPayload { break; case Event.Discover: case Event.Mutation: + case Event.BoxModel: event = layout(entry); break; case Event.Metadata: @@ -68,6 +69,9 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head case Event.Mutation: r.markup(entry.data, iframe); break; + case Event.BoxModel: + r.boxmodel(entry.data, iframe); + break; case Event.Resize: r.resize(entry.data[0], iframe); break; diff --git a/decode/layout.ts b/decode/layout.ts index 10758ddd..26861acd 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -1,6 +1,6 @@ import { resolve } from "../src/data/token"; import { Event, IDecodedEvent, Token } from "../types/data"; -import { IDecodedNode, IDocumentSize } from "../types/layout"; +import { IBoxModel, IDecodedNode, IDocumentSize } from "../types/layout"; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; @@ -11,6 +11,12 @@ export default function(tokens: Token[]): IDecodedEvent { let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; decoded.data.push(d); return decoded; + case Event.BoxModel: + for (let i = 2; i < tokens.length; i += 2) { + let boxmodel: IBoxModel = { id: tokens[i] as number, box: tokens[i + 1] as number[] }; + decoded.data.push(boxmodel); + } + return decoded; case Event.Discover: case Event.Mutation: let lastType = null; @@ -64,7 +70,6 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { tag: node[tagIndex] }; let hasAttribute = false; - let layouts = []; let attributes = {}; let value = null; for (let i = tagIndex + 1; i < node.length; i++) { @@ -76,12 +81,6 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { } else if (output.tag !== "*T" && keyIndex > 0) { hasAttribute = true; attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); - } else if (parts.length === 4) { - let layout = []; - for (let part of parts) { - layout.push(parseInt(part, 36)); - } - layouts.push(layout); } else if (output.tag === "*T") { value = token; if (parts.length === 2) { @@ -91,23 +90,22 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { if (wordCount > 0 && textCount === 0) { value = " "; } else if (wordCount === 0 && textCount > 0) { - value = Array(textCount).join("*"); + value = Array(textCount + 1).join("x"); } else if (wordCount > 0 && textCount > 0) { value = ""; let avg = Math.floor(textCount / wordCount); while (value.length < textCount + wordCount) { let gap = Math.min(avg, textCount + wordCount - value.length); - value += Array(gap).join("x") + " "; + value += Array(gap + 1).join("x") + " "; } } else { - value = Array(Math.floor((textCount + 1) / 2)).join("x "); + value = Array(Math.floor((textCount + 1) / 2) + 1).join("x "); } } } } } - if (layouts.length > 0) { output.layout = layouts; } if (hasAttribute) { output.attributes = attributes; } if (value) { output.value = value; } diff --git a/decode/metric.ts b/decode/metric.ts index 12441e4f..cb6acdc3 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -3,16 +3,19 @@ import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metric" let map: IMetricMap = {}; -map[Metric.NodeCount] = { name: "Node Count", unit: ""}; -map[Metric.ByteCount] = { name: "Byte Count", unit: "KB"}; -map[Metric.MutationCount] = { name: "Mutation Count", unit: ""}; -map[Metric.InteractionCount] = { name: "Interaction Count", unit: ""}; -map[Metric.ClickCount] = { name: "Click Count", unit: ""}; -map[Metric.ScriptErrorCount] = { name: "Script Errors", unit: ""}; -map[Metric.ImageErrorCount] = { name: "Image Errors", unit: ""}; +map[Metric.Nodes] = { name: "Node Count", unit: ""}; +map[Metric.Bytes] = { name: "Byte Count", unit: "KB"}; +map[Metric.StreamOneBytes] = { name: "Stream One Bytes", unit: "KB"}; +map[Metric.StreamTwoBytes] = { name: "Stream Two Bytes", unit: "KB"}; +map[Metric.Mutations] = { name: "Mutation Count", unit: ""}; +map[Metric.Interactions] = { name: "Interaction Count", unit: ""}; +map[Metric.Clicks] = { name: "Click Count", unit: ""}; +map[Metric.ScriptErrors] = { name: "Script Errors", unit: ""}; +map[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; map[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; map[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; -map[Metric.WireupLag] = { name: "Wireup Delay", unit: "ms"}; +map[Metric.BoxModelTime] = { name: "Box Model Time", unit: "ms"}; +map[Metric.WireupTime] = { name: "Wireup Delay", unit: "s"}; map[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; map[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px", value: "max"}; map[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px", value: "max"}; diff --git a/decode/render.ts b/decode/render.ts index adb87e78..c06e720e 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ import { IResize, IScroll, Scroll } from "../types/interaction"; -import { IDecodedNode } from "../types/layout"; +import { IBoxModel, IDecodedNode } from "../types/layout"; import { IDecodedMetric, IMetricMapValue } from "../types/metric"; let nodes = {}; @@ -39,6 +39,7 @@ export function metrics(data: IDecodedMetric, header: HTMLElement): void { function value(input: number, unit: string): number { switch (unit) { case "KB": return Math.round(input / 1024); + case "s": return Math.round(input / 1000); default: return input; } } @@ -48,6 +49,16 @@ function metricBox(metric: number, map: IMetricMapValue, metadata: string = null return `

  • ${metric}${map.unit}
    ${metadata}

    ${map.name}
    • `; } +export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { + for (let bm of data) { + let el = element(bm.id) as HTMLElement; + if (el && false) { + el.style.width = bm.box[2] + "px"; + el.style.height = bm.box[3] + "px"; + } + } +} + export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; for (let node of data) { diff --git a/src/core/config.ts b/src/core/config.ts index 93b8ba09..f1ee7a02 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,10 +1,10 @@ import { IConfig } from "@clarity-types/core"; let config: IConfig = { - longtask: 50, - lookahead: 250, + longtask: 30, + lookahead: 500, distance: 20, - delay: 250, + delay: 1000, showText: false, cssRules: false, tokens: [], diff --git a/src/core/task.ts b/src/core/task.ts index 33ea4e81..8a081a28 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -6,6 +6,13 @@ import * as metrics from "@src/metric"; let tracker: ITask = {}; let threshold = config.longtask; +// Debug: Start +let debug = {}; +debug[Metric.DiscoverTime] = "Discover"; +debug[Metric.MutationTime] = "Mutation"; +debug[Metric.BoxModelTime] = "BoxModel"; +// Debug: End + export function longtask(method: Metric): boolean { let elapsed = Date.now() - tracker[method]; return (elapsed > threshold); @@ -21,7 +28,7 @@ export function start(method: Metric): void { export function stop(method: Metric): void { let end = Date.now(); let duration = end - tracker[method]; - metrics.measure(method, duration); + metrics.counter(method, duration); } export async function idle(method: Metric): Promise { diff --git a/src/data/encode.ts b/src/data/encode.ts index b7414fc1..29fb07ee 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -8,7 +8,7 @@ export default function(envelope: boolean = false): Token[] { let t = time(); let tokens: Token[] = envelope ? [] : [t, Event.Metadata]; - metric.measure(Metric.WireupLag, t); + if (!envelope) { metric.counter(Metric.WireupTime, t); } tokens.push(metadata.sequence); tokens.push(metadata.version); diff --git a/src/data/upload.ts b/src/data/upload.ts index 947a21c1..54a8dd41 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -3,19 +3,20 @@ import { Metric } from "@clarity-types/metric"; import * as decode from "@decode/clarity"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; -import { measure } from "@src/metric"; +import { counter } from "@src/metric"; import metrics from "@src/metric/encode"; export default function(events: IEventQueue): void { - let payload: IPayload = { - e: envelope(), - m: metrics(), - a: events.one, - b: events.two - }; let upload = config.upload ? config.upload : send; - let data = JSON.stringify(payload); - measure(Metric.ByteCount, data.length); + let payload: IPayload = { e: envelope(), m: metrics(), a: events.one, b: events.two }; + let e = JSON.stringify(payload.e); + let m = JSON.stringify(payload.m); + let a = JSON.stringify(payload.a); + let b = JSON.stringify(payload.b); + let data = `{"e":${e},"m":${m},"a":${a},"b":${b}}`; + counter(Metric.Bytes, data.length); + counter(Metric.StreamOneBytes, a.length); + counter(Metric.StreamTwoBytes, b.length); upload(data); } diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts index dcc5f1fa..2b13f726 100644 --- a/src/diagnostic/encode.ts +++ b/src/diagnostic/encode.ts @@ -17,7 +17,7 @@ export default function(type: Event): Token[] { tokens.push(e.line); tokens.push(e.column); tokens.push(e.stack); - metric.counter(Metric.ScriptErrorCount); + metric.counter(Metric.ScriptErrors); } script.reset(); break; @@ -26,7 +26,7 @@ export default function(type: Event): Token[] { for (let e of images) { tokens.push(e.source); tokens.push(e.target); - metric.counter(Metric.ImageErrorCount); + metric.counter(Metric.ImageErrors); } image.reset(); break; diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts new file mode 100644 index 00000000..b964dff6 --- /dev/null +++ b/src/layout/boxmodel.ts @@ -0,0 +1,110 @@ +import { Event, Token } from "@clarity-types/data"; +import { IBoxModel } from "@clarity-types/layout"; +import { Metric } from "@clarity-types/metric"; +import config from "@src/core/config"; +import * as task from "@src/core/task"; +import queue from "@src/data/queue"; +import encode from "@src/layout/encode"; +import * as dom from "./dom"; + +let bm: {[key: number]: IBoxModel} = {}; +let updates: number[] = []; +let timeout: number = null; + +export function compute(): void { + if (timeout) { clearTimeout(timeout); } + timeout = window.setTimeout(schedule, config.lookahead); +} + +function schedule(): void { + boxmodel().then((data: Token[]) => { + queue(data); + }); +} + +async function boxmodel(): Promise { + let timer = Metric.BoxModelTime; + task.start(timer); + let values = dom.getLeafNodes(); + let doc = document.documentElement; + let x = "pageXOffset" in window ? window.pageXOffset : doc.scrollLeft; + let y = "pageYOffset" in window ? window.pageYOffset : doc.scrollTop; + + for (let value of values) { + if (task.longtask(timer)) { + await task.idle(timer); + x = "pageXOffset" in window ? window.pageXOffset : doc.scrollLeft; + y = "pageYOffset" in window ? window.pageYOffset : doc.scrollTop; + } + update(value.id, getLayout(x, y, dom.getNode(value.id) as Element)); + } + + let data = await encode(Event.BoxModel); + task.stop(timer); + return data; +} + +export function summarize(): IBoxModel[] { + let summary = []; + for (let id of updates) { + summary.push(bm[id]); + } + updates = []; + return summary; +} +function update(id: number, box: number[]): void { + let changed = true; + if (id in bm) { + changed = box.length === bm[id].box.length ? false : true; + if (changed === false) { + for (let i = 0; i < box.length; i++) { + if (box[i] !== bm[id].box[i]) { + changed = true; + break; + } + } + } + } + + if (changed) { + if (updates.indexOf(id) === -1) { updates.push(id); } + bm[id] = {id, box}; + } +} + +/* function getTextLayout(x: number, y: number, textNode: Node): number[] { + let layout: number[] = []; + let range = document.createRange(); + range.selectNodeContents(textNode); + let rects = range.getClientRects(); + + for (let i = 0; i < rects.length; i++) { + let rect = rects[i]; + layout.push(Math.floor(rect.left + x)); + layout.push(Math.floor(rect.top + y)); + layout.push(Math.floor(rect.width)); + layout.push(Math.floor(rect.height)); + } + + return layout.length > 0 ? layout : [0, 0, 0, 0]; +} */ + +function getLayout(x: number, y: number, element: Element): number[] { + let layout: number[] = [0, 0, 0, 0]; + let rect = element.getBoundingClientRect(); + + if (rect && rect.width > 0 && rect.height > 0) { + // getBoundingClientRect returns relative positioning to viewport and therefore needs + // addition of window scroll position to get position relative to document + // Also: using Math.floor() instead of Math.round() below because in Edge, + // getBoundingClientRect returns partial pixel values (e.g. 162.5px) and Chrome already + // floors the value (e.g. 162px). Keeping behavior consistent across + layout = [ + Math.floor(rect.left + x), + Math.floor(rect.top + y), + Math.floor(rect.width), + Math.floor(rect.height) + ]; + } + return layout; +} diff --git a/src/layout/discover.ts b/src/layout/discover.ts index 72c09c02..89330593 100644 --- a/src/layout/discover.ts +++ b/src/layout/discover.ts @@ -3,6 +3,7 @@ import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; +import * as boxmodel from "@src/layout/boxmodel"; import * as doc from "@src/layout/document"; import encode from "@src/layout/encode"; @@ -11,8 +12,9 @@ import processNode from "./node"; export function start(): void { discover().then((data: Token[]) => { doc.compute(); + boxmodel.compute(); queue(data); - }); + }); } async function discover(): Promise { diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 8ea3f5cc..a3effebe 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -10,10 +10,6 @@ let values: INodeValue[] = []; let updates: number[] = []; let changes: INodeChange[][] = []; -let backupIndex: number; -let backupNodes: Node[]; -let backupValues: INodeValue[]; - export function reset(): void { index = 1; nodes = []; @@ -47,8 +43,11 @@ export function add(node: Node, data: INodeData, source: Source): void { parent: parentId, next: nextId, children: [], - data + data, + active: true, + leaf: false }; + leaf(data.tag, id, parentId); track(id, source); } @@ -78,7 +77,7 @@ export function update(node: Node, data: INodeData, source: Source): void { } } else { // Mark this element as deleted if the parent has been updated to null - value["active"] = false; + value.active = false; } // Remove reference to this node from the old parent @@ -96,6 +95,7 @@ export function update(node: Node, data: INodeData, source: Source): void { value["data"][key] = data[key]; } } + leaf(data.tag, id, parentId); track(id, source); } } @@ -116,6 +116,16 @@ export function has(node: Node): boolean { return getId(node) in nodes; } +export function getLeafNodes(): INodeValue[] { + let v = []; + for (let id in values) { + if (values[id].active && values[id].leaf) { + v.push(values[id]); + } + } + return v; +} + export function summarize(): INodeValue[] { let v = []; for (let id of updates) { @@ -127,16 +137,17 @@ export function summarize(): INodeValue[] { return v; } -export function backup(): void { - backupNodes = Array.from(nodes); - backupValues = copy(values); - backupIndex = index; -} - -export function rollback(): void { - nodes = Array.from(backupNodes); - values = copy(backupValues); - index = backupIndex; +function leaf(tag: string, id: number, parentId: number): void { + if (id !== null && parentId !== null) { + switch (tag) { + case "*T": + values[parentId].leaf = true; + break; + case "SVG": + values[id].leaf = true; + break; + } + } } function getNextId(node: Node): number { diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 4c0e29a9..2d6a18ff 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -7,8 +7,9 @@ import time from "@src/core/time"; import hash from "@src/data/hash"; import {check} from "@src/data/token"; import * as metric from "@src/metric"; +import * as boxmodel from "./boxmodel"; import {doc} from "./document"; -import * as nodes from "./dom"; +import * as dom from "./dom"; window["HASH"] = hash; @@ -23,9 +24,16 @@ export default async function(type: Event): Promise { metric.measure(Metric.DocumentWidth, d.width); metric.measure(Metric.DocumentHeight, d.height); return tokens; + case Event.BoxModel: + let bm = boxmodel.summarize(); + for (let value of bm) { + tokens.push(value.id); + tokens.push(value.box); + } + return tokens; case Event.Discover: case Event.Mutation: - let values = nodes.summarize(); + let values = dom.summarize(); for (let value of values) { if (task.longtask(timer)) { await task.idle(timer); } let metadata = []; @@ -36,7 +44,7 @@ export default async function(type: Event): Promise { if (data[key]) { switch (key) { case "tag": - metric.counter(Metric.NodeCount); + metric.counter(Metric.Nodes); tokens.push(value.id); if (value.parent) { tokens.push(value.parent); } if (value.next) { tokens.push(value.next); } @@ -58,9 +66,9 @@ export default async function(type: Event): Promise { } break; case "value": - let parent = nodes.getNode(value.parent); + let parent = dom.getNode(value.parent); if (parent === null) { console.warn("Unexpected | Node value: " + JSON.stringify(value)); } - let parentTag = nodes.get(parent) ? nodes.get(parent).data.tag : null; + let parentTag = dom.get(parent) ? dom.get(parent).data.tag : null; let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; metadata.push(text(tag, data[key])); break; diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index 36b24d54..90da6bdb 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -3,6 +3,7 @@ import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; +import * as boxmodel from "@src/layout/boxmodel"; import * as doc from "@src/layout/document"; import encode from "@src/layout/encode"; import processNode from "./node"; @@ -25,6 +26,7 @@ export function end(): void { function handle(mutations: MutationRecord[]): void { process(mutations).then((data: Token[]) => { doc.compute(); + boxmodel.compute(); queue(data); }); } diff --git a/src/layout/node.ts b/src/layout/node.ts index b687fd63..8b700c97 100644 --- a/src/layout/node.ts +++ b/src/layout/node.ts @@ -25,7 +25,6 @@ export default function(node: Node, source: Source): void { let parent = node.parentElement; if (call === "update" || (parent && dom.has(parent) && parent.tagName !== "STYLE")) { let textData = { tag: "*T", value: node.nodeValue }; - textData["layout"] = getTextLayout(node); dom[call](node, textData, source); } break; @@ -50,7 +49,6 @@ export default function(node: Node, source: Source): void { break; default: let data = { tag, attributes: getAttributes(element.attributes) }; - data["layout"] = getLayout(element); dom[call](node, data, source); break; } @@ -96,48 +94,3 @@ function getAttributes(attributes: NamedNodeMap): {[key: string]: string} { } return output; } - -function getTextLayout(textNode: Node): number[] { - let layout: number[] = []; - let range = document.createRange(); - range.selectNodeContents(textNode); - let rects = range.getClientRects(); - let doc = document.documentElement; - for (let i = 0; i < rects.length; i++) { - let rect = rects[i]; - layout.push(Math.floor(rect.left) + ("pageXOffset" in window ? window.pageXOffset : doc.scrollLeft)); - layout.push(Math.floor(rect.top) + ("pageYOffset" in window ? window.pageYOffset : doc.scrollTop)); - layout.push(Math.round(rect.width)); - layout.push(Math.round(rect.height)); - } - return layout; -} - -function getLayout(element: Element): number[] { - // In IE, calling getBoundingClientRect on a node that is disconnected - // from a DOM tree, sometimes results in a 'Unspecified Error' - // Wrapping this in try/catch is faster than checking whether element is connected to DOM - let layout: number[] = []; - let rect = null; - let doc = document.documentElement; - try { - rect = element.getBoundingClientRect(); - } catch (e) { - // Ignore - } - - if (rect && rect.width > 0 && rect.height > 0) { - // getBoundingClientRect returns relative positioning to viewport and therefore needs - // addition of window scroll position to get position relative to document - // Also: using Math.floor() instead of Math.round() below because in Edge, - // getBoundingClientRect returns partial pixel values (e.g. 162.5px) and Chrome already - // floors the value (e.g. 162px). Keeping behavior consistent across - layout = [ - Math.floor(rect.left) + ("pageXOffset" in window ? window.pageXOffset : doc.scrollLeft), - Math.floor(rect.top) + ("pageYOffset" in window ? window.pageYOffset : doc.scrollTop), - Math.round(rect.width), - Math.round(rect.height) - ]; - } - return layout; -} diff --git a/types/data.d.ts b/types/data.d.ts index 4805f23b..8de831bd 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -7,6 +7,7 @@ export const enum Event { Metadata, Discover, Mutation, + BoxModel, Mouse, Touch, Keyboard, diff --git a/types/layout.d.ts b/types/layout.d.ts index 2c1bfc65..fc05e9f6 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -13,7 +13,6 @@ export interface IAttributes { export interface INodeData { tag: string; attributes?: IAttributes; - layout?: number[]; value?: string; } @@ -23,6 +22,9 @@ export interface INodeValue { next: number; children: number[]; data: INodeData; + /* Metadata */ + active: boolean; + leaf: boolean; } export interface INodeChange { @@ -37,7 +39,6 @@ export interface IDecodedNode { next: number; tag: string; attributes?: IAttributes; - layout?: number[][]; value?: string; } @@ -45,3 +46,8 @@ export interface IDocumentSize { width: number; height: number; } + +export interface IBoxModel { + id: number; + box: number[]; +} diff --git a/types/metric.d.ts b/types/metric.d.ts index 56a1ac60..777bd349 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -8,18 +8,21 @@ export const enum MetricType { export const enum Metric { /* Counter */ - NodeCount, - ByteCount, - MutationCount, - InteractionCount, - ClickCount, - ScriptErrorCount, - ImageErrorCount, - /* Summary */ + Nodes, + Bytes, + StreamOneBytes, + StreamTwoBytes, + Mutations, + Interactions, + Clicks, + ScriptErrors, + ImageErrors, DiscoverTime, MutationTime, - WireupLag, + BoxModelTime, + WireupTime, ActiveTime, + /* Summary */ ViewportWidth, ViewportHeight, DocumentWidth, From 26181258259a3c4e1aa18c7df6d75c7a37a25c8b Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 26 Aug 2019 09:15:29 -0700 Subject: [PATCH 058/105] Simplifying SVG handling --- decode/render.ts | 15 ++++++--------- src/layout/dom.ts | 2 +- src/layout/node.ts | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index c06e720e..5f5a4131 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -128,15 +128,8 @@ export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { } function createElement(doc: Document, tag: string, parent: HTMLElement): HTMLElement { - if (tag === "svg") { + if (tag && tag.indexOf("s:") === 0) { return doc.createElementNS(svgns, tag) as HTMLElement; - } else { - while (parent && parent.tagName !== "BODY") { - if (parent.tagName === "svg") { - return doc.createElementNS(svgns, tag) as HTMLElement; - } - parent = parent.parentElement; - } } return doc.createElement(tag); } @@ -174,7 +167,11 @@ function setAttributes(node: HTMLElement, attributes: object): void { for (let attribute in attributes) { if (attributes[attribute] !== undefined) { try { - node.setAttribute(attribute, attributes[attribute]); + if (attribute.indexOf("xlink:") === 0) { + node.setAttributeNS("http://www.w3.org/1999/xlink", attribute, attributes[attribute]); + } else { + node.setAttribute(attribute, attributes[attribute]); + } } catch (ex) { console.warn("Node: " + node + " | " + JSON.stringify(attributes)); console.warn("Exception encountered while adding attributes: " + ex); diff --git a/src/layout/dom.ts b/src/layout/dom.ts index a3effebe..aae19fec 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -143,7 +143,7 @@ function leaf(tag: string, id: number, parentId: number): void { case "*T": values[parentId].leaf = true; break; - case "SVG": + case "s:svg": values[id].leaf = true; break; } diff --git a/src/layout/node.ts b/src/layout/node.ts index 8b700c97..486b7a88 100644 --- a/src/layout/node.ts +++ b/src/layout/node.ts @@ -31,6 +31,7 @@ export default function(node: Node, source: Source): void { case Node.ELEMENT_NODE: let element = (node as HTMLElement); let tag = element.tagName; + tag = (element.namespaceURI === "http://www.w3.org/2000/svg") ? "s:" + tag : tag; switch (tag) { case "SCRIPT": case "NOSCRIPT": From e781f182e043b0740bb160128a49210561fced9a Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 26 Aug 2019 16:36:24 -0700 Subject: [PATCH 059/105] Bug fix related to style mutations --- decode/render.ts | 2 +- src/layout/dom.ts | 2 +- src/layout/node.ts | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index 5f5a4131..8316d56d 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -128,7 +128,7 @@ export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { } function createElement(doc: Document, tag: string, parent: HTMLElement): HTMLElement { - if (tag && tag.indexOf("s:") === 0) { + if (tag && tag.indexOf("svg:") === 0) { return doc.createElementNS(svgns, tag) as HTMLElement; } return doc.createElement(tag); diff --git a/src/layout/dom.ts b/src/layout/dom.ts index aae19fec..01a987d4 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -143,7 +143,7 @@ function leaf(tag: string, id: number, parentId: number): void { case "*T": values[parentId].leaf = true; break; - case "s:svg": + case "svg:svg": values[id].leaf = true; break; } diff --git a/src/layout/node.ts b/src/layout/node.ts index 486b7a88..34eb8830 100644 --- a/src/layout/node.ts +++ b/src/layout/node.ts @@ -8,6 +8,14 @@ export default function(node: Node, source: Source): void { // Do not track this change if we are attempting to remove a node before discovering it if (source === Source.ChildListRemove && dom.has(node) === false) { return; } + // Special handling for text nodes that belong to style nodes + if (source !== Source.Discover && + node.nodeType === Node.TEXT_NODE && + node.parentElement && + node.parentElement.tagName === "STYLE") { + node = node.parentNode; + } + let call = dom.has(node) ? "update" : "add"; switch (node.nodeType) { case Node.DOCUMENT_TYPE_NODE: @@ -31,7 +39,7 @@ export default function(node: Node, source: Source): void { case Node.ELEMENT_NODE: let element = (node as HTMLElement); let tag = element.tagName; - tag = (element.namespaceURI === "http://www.w3.org/2000/svg") ? "s:" + tag : tag; + tag = (element.namespaceURI === "http://www.w3.org/2000/svg") ? "svg:" + tag : tag; switch (tag) { case "SCRIPT": case "NOSCRIPT": From e37779e57d74895efbe6f8f66f7a90c9393e6a75 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 27 Aug 2019 08:23:29 -0700 Subject: [PATCH 060/105] Serializing async call execution --- src/core/task.ts | 37 +++++++++++++++++++++++++++++-------- src/layout/boxmodel.ts | 8 +++++--- src/layout/discover.ts | 12 +++++++----- src/layout/mutation.ts | 24 ++++++++++++++---------- types/core.d.ts | 12 ++++++++++-- 5 files changed, 65 insertions(+), 28 deletions(-) diff --git a/src/core/task.ts b/src/core/task.ts index 8a081a28..20d403c0 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,17 +1,38 @@ -import {ITask } from "@clarity-types/core"; +import { IAsyncTask, ITaskTiming, TaskCallback, TaskFunction } from "@clarity-types/core"; +import {Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as metrics from "@src/metric"; -let tracker: ITask = {}; +let tracker: ITaskTiming = {}; let threshold = config.longtask; +let queue: IAsyncTask[] = []; +let active: IAsyncTask = null; -// Debug: Start -let debug = {}; -debug[Metric.DiscoverTime] = "Discover"; -debug[Metric.MutationTime] = "Mutation"; -debug[Metric.BoxModelTime] = "BoxModel"; -// Debug: End +export async function schedule(task: TaskFunction, callback: TaskCallback): Promise { + // If this task is already scheduled, skip it + for (let q of queue) { + if (q.task === task) { + return; + } + } + + // Otherwise, add thit to the queue + queue.push({task, callback}); + if (active === null) { run(); } +} + +function run(): void { + let entry = queue.shift(); + if (entry) { + active = entry; + entry.task().then((data: Token[]) => { + entry.callback(data); + active = null; + run(); + }); + } +} export function longtask(method: Metric): boolean { let elapsed = Date.now() - tracker[method]; diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index b964dff6..c22a8999 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -17,9 +17,11 @@ export function compute(): void { } function schedule(): void { - boxmodel().then((data: Token[]) => { - queue(data); - }); + task.schedule(boxmodel, done); +} + +function done(data: Token[]): void { + queue(data); } async function boxmodel(): Promise { diff --git a/src/layout/discover.ts b/src/layout/discover.ts index 89330593..5bb2e566 100644 --- a/src/layout/discover.ts +++ b/src/layout/discover.ts @@ -10,11 +10,13 @@ import encode from "@src/layout/encode"; import processNode from "./node"; export function start(): void { - discover().then((data: Token[]) => { - doc.compute(); - boxmodel.compute(); - queue(data); - }); + task.schedule(discover, done); +} + +function done(data: Token[]): void { + doc.compute(); + boxmodel.compute(); + queue(data); } async function discover(): Promise { diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index 90da6bdb..6169d021 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -9,6 +9,7 @@ import encode from "@src/layout/encode"; import processNode from "./node"; let observer: MutationObserver; +let mutations: MutationRecord[] = []; export function start(): void { if (observer) { @@ -23,20 +24,23 @@ export function end(): void { observer = null; } -function handle(mutations: MutationRecord[]): void { - process(mutations).then((data: Token[]) => { - doc.compute(); - boxmodel.compute(); - queue(data); - }); +function handle(m: MutationRecord[]): void { + // Queue up mutation records for asynchronous processing + for (let i = 0; i < m.length; i++) { mutations.push(m[i]); } + task.schedule(process, done); } -async function process(mutations: MutationRecord[]): Promise { +function done(data: Token[]): void { + doc.compute(); + boxmodel.compute(); + queue(data); +} + +async function process(): Promise { let timer = Metric.MutationTime; task.start(timer); - let length = mutations.length; - for (let i = 0; i < length; i++) { - let mutation = mutations[i]; + while (mutations.length > 0) { + let mutation = mutations.shift(); let target = mutation.target; switch (mutation.type) { diff --git a/types/core.d.ts b/types/core.d.ts index 6e8b58a5..4aa1faee 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -1,4 +1,7 @@ -import { IPayload } from "./data"; +import { IPayload, Token } from "./data"; + +type TaskFunction = () => Promise; +type TaskCallback = (data: Token[]) => void; export interface IEventBindingPair { target: EventTarget; @@ -28,6 +31,11 @@ export const enum Task { Active } -export interface ITask { +export interface ITaskTiming { [key: number]: number; } + +export interface IAsyncTask { + task: TaskFunction; + callback: TaskCallback; +} From 8325c1fe6c7f290f7ca394ae6a3e3970293e4bd2 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 27 Aug 2019 11:24:24 -0700 Subject: [PATCH 061/105] Adding upload support & getting rid of two streams --- decode/clarity.ts | 10 +--------- decode/metric.ts | 2 -- src/core/config.ts | 1 + src/data/queue.ts | 24 ++++-------------------- src/data/upload.ts | 20 ++++++++++---------- types/core.d.ts | 1 + types/data.d.ts | 8 +------- types/metric.d.ts | 2 -- 8 files changed, 18 insertions(+), 50 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index b64e3de0..dfe8e197 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -23,7 +23,7 @@ export function json(data: string): IDecodedPayload { r.reset(); } - let encoded: Token[][] = merge(payload.a, payload.b); + let encoded: Token[][] = payload.d; payloads.push(payload); for (let entry of encoded) { @@ -81,11 +81,3 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head } } } - -function merge(one: Token[][], two: Token[][]): Token[][] { - return one.concat(two).sort(function(a: Token[], b: Token[]): number { - let x = a[0] as number; - let y = b[0] as number; - return x - y; - }); -} diff --git a/decode/metric.ts b/decode/metric.ts index cb6acdc3..27e538c0 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -5,8 +5,6 @@ let map: IMetricMap = {}; map[Metric.Nodes] = { name: "Node Count", unit: ""}; map[Metric.Bytes] = { name: "Byte Count", unit: "KB"}; -map[Metric.StreamOneBytes] = { name: "Stream One Bytes", unit: "KB"}; -map[Metric.StreamTwoBytes] = { name: "Stream Two Bytes", unit: "KB"}; map[Metric.Mutations] = { name: "Mutation Count", unit: ""}; map[Metric.Interactions] = { name: "Interaction Count", unit: ""}; map[Metric.Clicks] = { name: "Click Count", unit: ""}; diff --git a/src/core/config.ts b/src/core/config.ts index f1ee7a02..7ed93558 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -8,6 +8,7 @@ let config: IConfig = { showText: false, cssRules: false, tokens: [], + url: "", upload: null }; diff --git a/src/data/queue.ts b/src/data/queue.ts index d1ab779d..e2105516 100644 --- a/src/data/queue.ts +++ b/src/data/queue.ts @@ -1,30 +1,14 @@ -import { Event, Flush, IEventQueue, Token } from "@clarity-types/data"; +import { Flush, Token } from "@clarity-types/data"; import config from "@src/core/config"; import upload from "@src/data/upload"; -let events: IEventQueue = { one: [], two: [] }; +let events: Token[][] = []; let timeout: number = null; window["PAYLOAD"] = []; export default function(data: Token[], flush: Flush = Flush.Schedule): void { - let event = data[1]; - - switch (event) { - case Event.Mouse: - case Event.Touch: - case Event.Keyboard: - case Event.Selection: - case Event.Resize: - case Event.Scroll: - case Event.Document: - case Event.Visibility: - events.one.push(data); - break; - default: - events.two.push(data); - break; - } + events.push(data); switch (flush) { case Flush.Schedule: @@ -44,5 +28,5 @@ function dequeue(): void { } function reset(): void { - events = { one: [], two: [] }; + events = []; } diff --git a/src/data/upload.ts b/src/data/upload.ts index 54a8dd41..601e202f 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,25 +1,25 @@ -import { IEventQueue, IPayload } from "@clarity-types/data"; +import { IPayload, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; -import * as decode from "@decode/clarity"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; import { counter } from "@src/metric"; import metrics from "@src/metric/encode"; -export default function(events: IEventQueue): void { +export default function(events: Token[][]): void { let upload = config.upload ? config.upload : send; - let payload: IPayload = { e: envelope(), m: metrics(), a: events.one, b: events.two }; + let payload: IPayload = { e: envelope(), m: metrics(), d: events }; let e = JSON.stringify(payload.e); let m = JSON.stringify(payload.m); - let a = JSON.stringify(payload.a); - let b = JSON.stringify(payload.b); - let data = `{"e":${e},"m":${m},"a":${a},"b":${b}}`; + let d = JSON.stringify(payload.d); + let data = `{"e":${e},"m":${m},"d":${d}}`; counter(Metric.Bytes, data.length); - counter(Metric.StreamOneBytes, a.length); - counter(Metric.StreamTwoBytes, b.length); upload(data); } function send(data: string): void { - decode.json(data); + if (config.url.length > 0) { + let xhr = new XMLHttpRequest(); + xhr.open("POST", config.url); + xhr.send(data); + } } diff --git a/types/core.d.ts b/types/core.d.ts index 4aa1faee..bf25b870 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -20,6 +20,7 @@ export interface IConfig { showText?: boolean; cssRules?: boolean; tokens?: string[]; + url?: string; upload?: (data: string) => void; } diff --git a/types/data.d.ts b/types/data.d.ts index 8de831bd..4ac0e7d9 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -28,16 +28,10 @@ export const enum Flush { None } -export interface IEventQueue { - one: Token[][]; - two: Token[][]; -} - export interface IPayload { e: Token[]; m: Token[]; - a: Token[][]; - b: Token[][]; + d: Token[][]; } export interface IDecodedPayload { diff --git a/types/metric.d.ts b/types/metric.d.ts index 777bd349..ea8e1154 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -10,8 +10,6 @@ export const enum Metric { /* Counter */ Nodes, Bytes, - StreamOneBytes, - StreamTwoBytes, Mutations, Interactions, Clicks, From 417e515489cd6a9ca1014afc6528be7696e7898e Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 27 Aug 2019 14:24:49 -0700 Subject: [PATCH 062/105] Ability to set ids via configurations --- src/core/config.ts | 3 +++ src/data/metadata.ts | 7 ++++--- types/core.d.ts | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/config.ts b/src/core/config.ts index 7ed93558..ecde9d7d 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,6 +1,9 @@ import { IConfig } from "@clarity-types/core"; let config: IConfig = { + pageId: null, + userId: null, + projectId: null, longtask: 30, lookahead: 500, distance: 20, diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 34466565..75c9271e 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,4 +1,5 @@ import { Flush, IMetadata, Token } from "@clarity-types/data"; +import config from "@src/core/config"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; @@ -10,9 +11,9 @@ export function start(): void { metadata = { sequence: 0, version, - pageId: guid(), - userId: guid(), - projectId: hash(location.host), + pageId: config.pageId || guid(), + userId: config.userId || guid(), + projectId: config.projectId || hash(location.host), url: location.href, title: document.title, referrer: document.referrer diff --git a/types/core.d.ts b/types/core.d.ts index bf25b870..90a3a1b4 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -13,6 +13,9 @@ export interface IBindingContainer { } export interface IConfig { + pageId?: string; + userId?: string; + projectId?: string; longtask?: number; lookahead?: number; distance?: number; From d08d51e731a2fa7d544f3d4080758f95f9203e41 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 28 Aug 2019 07:16:47 -0700 Subject: [PATCH 063/105] Bugfix: Maintaining order during complex mutations --- src/layout/dom.ts | 10 +++++++++- src/layout/encode.ts | 1 - 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 01a987d4..58846b56 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -164,7 +164,15 @@ function copy(input: INodeValue[]): INodeValue[] { } function track(id: number, source: Source): void { - if (updates.indexOf(id) === -1) { updates.push(id); } + // Keep track of the order in which mutations happened, they may not be sequential + // Edge case: If an element is added later on, and pre-discovered element is moved as a child. + // In that case, we need to reorder the prediscovered element in the update list to keep visualization consistent. + let uIndex = updates.indexOf(id); + if (uIndex >= 0 && source === Source.ChildListAdd) { + updates.splice(uIndex, 1); + updates.push(id); + } else if (uIndex === -1) { updates.push(id); } + if (DEVTOOLS_HOOK in window) { let value = copy([values[id]])[0]; let change = { time: time(), source, value }; diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 2d6a18ff..9efd735b 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -67,7 +67,6 @@ export default async function(type: Event): Promise { break; case "value": let parent = dom.getNode(value.parent); - if (parent === null) { console.warn("Unexpected | Node value: " + JSON.stringify(value)); } let parentTag = dom.get(parent) ? dom.get(parent).data.tag : null; let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; metadata.push(text(tag, data[key])); From 75de7a8e05670c548115ae3b8f8365d4058f4665 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 28 Aug 2019 08:05:46 -0700 Subject: [PATCH 064/105] Adding support for insertRule and deleteRule --- src/layout/mutation.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index 6169d021..d3f74c62 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -10,6 +10,8 @@ import processNode from "./node"; let observer: MutationObserver; let mutations: MutationRecord[] = []; +let insertRule: (rule: string, index?: number) => number; +let deleteRule: (index?: number) => void; export function start(): void { if (observer) { @@ -17,11 +19,30 @@ export function start(): void { } observer = window["MutationObserver"] ? new MutationObserver(handle) : null; observer.observe(document, { attributes: true, childList: true, characterData: true, subtree: true }); + insertRule = CSSStyleSheet.prototype.insertRule; + deleteRule = CSSStyleSheet.prototype.deleteRule; + + // Some popular open source libraries, like styled-components, optimize performance + // by injecting CSS using insertRule API vs. appending text node. A side effect of + // using javascript API is that it doesn't trigger DOM mutation and therefore we + // need to override the insertRule API and listen for changes manually. + CSSStyleSheet.prototype.insertRule = function(rule: string, index?: number): number { + let value = insertRule.call(this, rule, index); + generate(this.ownerNode, "characterData"); + return value; + }; + + CSSStyleSheet.prototype.deleteRule = function(index?: number): void { + deleteRule.call(this, index); + generate(this.ownerNode, "characterData"); + }; } export function end(): void { observer.disconnect(); observer = null; + CSSStyleSheet.prototype.insertRule = insertRule; + CSSStyleSheet.prototype.deleteRule = deleteRule; } function handle(m: MutationRecord[]): void { @@ -79,3 +100,17 @@ async function process(): Promise { task.stop(timer); return data; } + +function generate(target: Node, type: MutationRecordType): void { + handle([{ + addedNodes: null, + attributeName: null, + attributeNamespace: null, + nextSibling: null, + oldValue: null, + previousSibling: null, + removedNodes: null, + target, + type + }]); +} From 9261b33163a6322605ae579df41a30b4060f1099 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 28 Aug 2019 08:34:37 -0700 Subject: [PATCH 065/105] Default to masking everything --- decode/layout.ts | 61 +++++++++++++++++++++++++++----------------- decode/render.ts | 2 +- src/core/config.ts | 1 - src/layout/encode.ts | 44 ++++++++++++++++++++++---------- types/core.d.ts | 1 - 5 files changed, 69 insertions(+), 40 deletions(-) diff --git a/decode/layout.ts b/decode/layout.ts index 26861acd..d86fe218 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -2,6 +2,8 @@ import { resolve } from "../src/data/token"; import { Event, IDecodedEvent, Token } from "../types/data"; import { IBoxModel, IDecodedNode, IDocumentSize } from "../types/layout"; +let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; + export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; let event = tokens[1] as Event; @@ -75,34 +77,22 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { for (let i = tagIndex + 1; i < node.length; i++) { let token = node[i] as string; let keyIndex = token.indexOf("="); - let parts = token.split("*"); if (i === (node.length - 1) && output.tag === "STYLE") { value = token; } else if (output.tag !== "*T" && keyIndex > 0) { hasAttribute = true; - attributes[token.substr(0, keyIndex)] = token.substr(keyIndex + 1); - } else if (output.tag === "*T") { - value = token; - if (parts.length === 2) { - let textCount = parseInt(parts[0], 36); - let wordCount = parseInt(parts[1], 36); - if (isFinite(textCount) && isFinite(wordCount)) { - if (wordCount > 0 && textCount === 0) { - value = " "; - } else if (wordCount === 0 && textCount > 0) { - value = Array(textCount + 1).join("x"); - } else if (wordCount > 0 && textCount > 0) { - value = ""; - let avg = Math.floor(textCount / wordCount); - while (value.length < textCount + wordCount) { - let gap = Math.min(avg, textCount + wordCount - value.length); - value += Array(gap + 1).join("x") + " "; - } - } else { - value = Array(Math.floor((textCount + 1) / 2) + 1).join("x "); - } - } + let k = token.substr(0, keyIndex); + let v = unmask(token.substr(keyIndex + 1)); + switch (k) { + case "src": + v = placeholderImage; + break; + default: + break; } + attributes[k] = v; + } else if (output.tag === "*T") { + value = unmask(token); } } @@ -111,3 +101,28 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { return output; } + +function unmask(value: string): string { + let parts = value.split("*"); + if (parts.length === 2) { + let textCount = parseInt(parts[0], 36); + let wordCount = parseInt(parts[1], 36); + if (isFinite(textCount) && isFinite(wordCount)) { + if (wordCount > 0 && textCount === 0) { + value = " "; + } else if (wordCount === 0 && textCount > 0) { + value = Array(textCount + 1).join("x"); + } else if (wordCount > 0 && textCount > 0) { + value = ""; + let avg = Math.floor(textCount / wordCount); + while (value.length < textCount + wordCount) { + let gap = Math.min(avg, textCount + wordCount - value.length); + value += Array(gap + 1).join("x") + " "; + } + } else { + value = Array(Math.floor((textCount + 1) / 2) + 1).join("x "); + } + } + } + return value; +} diff --git a/decode/render.ts b/decode/render.ts index 8316d56d..2b58cec2 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -52,7 +52,7 @@ function metricBox(metric: number, map: IMetricMapValue, metadata: string = null export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { for (let bm of data) { let el = element(bm.id) as HTMLElement; - if (el && false) { + if (el) { el.style.width = bm.box[2] + "px"; el.style.height = bm.box[3] + "px"; } diff --git a/src/core/config.ts b/src/core/config.ts index ecde9d7d..2e0e43cd 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -8,7 +8,6 @@ let config: IConfig = { lookahead: 500, distance: 20, delay: 1000, - showText: false, cssRules: false, tokens: [], url: "", diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 2d6a18ff..bffd20b7 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -1,7 +1,6 @@ import {Event, Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/layout"; import {Metric} from "@clarity-types/metric"; -import config from "@src/core/config"; import * as task from "@src/core/task"; import time from "@src/core/time"; import hash from "@src/data/hash"; @@ -53,7 +52,7 @@ export default async function(type: Event): Promise { case "attributes": for (let attr in data[key]) { if (data[key][attr] !== undefined) { - metadata.push(`${attr}=${data[key][attr]}`); + metadata.push(attribute(attr, data[key][attr])); } } break; @@ -97,25 +96,42 @@ function meta(metadata: string[]): string[] | string[][] { return check(hashed) && hashed.length < value.length ? [[hashed]] : metadata; } +function attribute(key: string, value: string): string { + switch (key) { + case "src": + case "title": + case "alt": + return `${key}=`; + case "value": + case "placeholder": + return `${key}=${mask(value)}`; + default: + return `${key}=${value}`; + } +} + function text(tag: string, value: string): string { switch (tag) { case "STYLE": case "TITLE": return value; default: - if (config.showText) { return value; } - let wasWhiteSpace = false; - let textCount = 0; - let wordCount = 0; - for (let i = 0; i < value.length; i++) { - let code = value.charCodeAt(i); - let isWhiteSpace = (code === 32 || code === 10 || code === 9 || code === 13); - textCount += isWhiteSpace ? 0 : 1; - wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; - wasWhiteSpace = isWhiteSpace; - } - return `${textCount.toString(36)}*${wordCount.toString(36)}`; + return mask(value); + } +} + +function mask(value: string): string { + let wasWhiteSpace = false; + let textCount = 0; + let wordCount = 0; + for (let i = 0; i < value.length; i++) { + let code = value.charCodeAt(i); + let isWhiteSpace = (code === 32 || code === 10 || code === 9 || code === 13); + textCount += isWhiteSpace ? 0 : 1; + wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; + wasWhiteSpace = isWhiteSpace; } + return `${textCount.toString(36)}*${wordCount.toString(36)}`; } function layout(l: number[]): string[] { diff --git a/types/core.d.ts b/types/core.d.ts index 90a3a1b4..32c148f9 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -20,7 +20,6 @@ export interface IConfig { lookahead?: number; distance?: number; delay?: number; - showText?: boolean; cssRules?: boolean; tokens?: string[]; url?: string; From 4820419ddcce0577e0af2c1a78d783c752145d3c Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 28 Aug 2019 08:38:14 -0700 Subject: [PATCH 066/105] Adding support for node level masking --- src/layout/dom.ts | 10 +++++++++- types/layout.d.ts | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 58846b56..ceb0f344 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -3,6 +3,8 @@ import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; const DEVTOOLS_HOOK: string = "__CLARITY_DEVTOOLS_HOOK__"; +const MASK_ATTRIBUTE = "data-clarity-mask"; +const UNMASK_ATTRIBUTE = "data-clarity-umask"; let index: number = 1; let nodes: Node[] = []; @@ -32,11 +34,16 @@ export function add(node: Node, data: INodeData, source: Source): void { let id = getId(node, true); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); + let mask = true; if (parentId >= 0 && values[parentId]) { values[parentId].children.push(id); + mask = values[parentId].mask; } + if (data.attributes && MASK_ATTRIBUTE in data.attributes) { mask = true; } + if (data.attributes && UNMASK_ATTRIBUTE in data.attributes) { mask = false; } + nodes[id] = node; values[id] = { id, @@ -45,7 +52,8 @@ export function add(node: Node, data: INodeData, source: Source): void { children: [], data, active: true, - leaf: false + leaf: false, + mask }; leaf(data.tag, id, parentId); track(id, source); diff --git a/types/layout.d.ts b/types/layout.d.ts index fc05e9f6..1952f223 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -25,6 +25,7 @@ export interface INodeValue { /* Metadata */ active: boolean; leaf: boolean; + mask: boolean; } export interface INodeChange { From b29015954f9ba4f6748da9df360bea6a06ddd2fd Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 28 Aug 2019 09:40:56 -0700 Subject: [PATCH 067/105] Adding support for unmasking attribute --- src/layout/dom.ts | 10 +++++----- src/layout/encode.ts | 14 +++++++------- types/layout.d.ts | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/layout/dom.ts b/src/layout/dom.ts index ceb0f344..51c0d052 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -34,15 +34,15 @@ export function add(node: Node, data: INodeData, source: Source): void { let id = getId(node, true); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); - let mask = true; + let masked = true; if (parentId >= 0 && values[parentId]) { values[parentId].children.push(id); - mask = values[parentId].mask; + masked = values[parentId].masked; } - if (data.attributes && MASK_ATTRIBUTE in data.attributes) { mask = true; } - if (data.attributes && UNMASK_ATTRIBUTE in data.attributes) { mask = false; } + if (data.attributes && MASK_ATTRIBUTE in data.attributes) { masked = true; } + if (data.attributes && UNMASK_ATTRIBUTE in data.attributes) { masked = false; } nodes[id] = node; values[id] = { @@ -53,7 +53,7 @@ export function add(node: Node, data: INodeData, source: Source): void { data, active: true, leaf: false, - mask + masked }; leaf(data.tag, id, parentId); track(id, source); diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 30d16bfe..69ba7578 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -52,7 +52,7 @@ export default async function(type: Event): Promise { case "attributes": for (let attr in data[key]) { if (data[key][attr] !== undefined) { - metadata.push(attribute(attr, data[key][attr])); + metadata.push(attribute(value.masked, attr, data[key][attr])); } } break; @@ -68,7 +68,7 @@ export default async function(type: Event): Promise { let parent = dom.getNode(value.parent); let parentTag = dom.get(parent) ? dom.get(parent).data.tag : null; let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; - metadata.push(text(tag, data[key])); + metadata.push(text(value.masked, tag, data[key])); break; } } @@ -95,27 +95,27 @@ function meta(metadata: string[]): string[] | string[][] { return check(hashed) && hashed.length < value.length ? [[hashed]] : metadata; } -function attribute(key: string, value: string): string { +function attribute(masked: boolean, key: string, value: string): string { switch (key) { case "src": case "title": case "alt": - return `${key}=`; + return `${key}=${masked ? "" : value}`; case "value": case "placeholder": - return `${key}=${mask(value)}`; + return `${key}=${masked ? mask(value) : value}`; default: return `${key}=${value}`; } } -function text(tag: string, value: string): string { +function text(masked: boolean, tag: string, value: string): string { switch (tag) { case "STYLE": case "TITLE": return value; default: - return mask(value); + return masked ? mask(value) : value; } } diff --git a/types/layout.d.ts b/types/layout.d.ts index 1952f223..8010bda9 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -25,7 +25,7 @@ export interface INodeValue { /* Metadata */ active: boolean; leaf: boolean; - mask: boolean; + masked: boolean; } export interface INodeChange { From 6d28ea3f76a754870b530bc7f513ffbefac69040 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 29 Aug 2019 06:58:48 -0700 Subject: [PATCH 068/105] Rendering mitigation and adding IMG as leaf node --- decode/render.ts | 6 ++++-- src/layout/dom.ts | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index 2b58cec2..9cbbbf4f 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -53,8 +53,10 @@ export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { for (let bm of data) { let el = element(bm.id) as HTMLElement; if (el) { - el.style.width = bm.box[2] + "px"; - el.style.height = bm.box[3] + "px"; + el.style.maxWidth = bm.box[2] + "px"; + el.style.minWidth = Math.max(bm.box[2] - 5, 0) + "px"; + el.style.maxHeight = bm.box[3] + "px"; + el.style.overflow = "hidden"; } } } diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 51c0d052..00698828 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -151,6 +151,7 @@ function leaf(tag: string, id: number, parentId: number): void { case "*T": values[parentId].leaf = true; break; + case "IMG": case "svg:svg": values[id].leaf = true; break; From 28ce87d7102a215295c6d59febd88711c6e720b3 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 30 Aug 2019 18:23:58 -0700 Subject: [PATCH 069/105] Layout rendering fixes --- decode/layout.ts | 7 ++++--- decode/render.ts | 1 + src/data/upload.ts | 1 + src/layout/dom.ts | 11 ++++++++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/decode/layout.ts b/decode/layout.ts index d86fe218..e91cc485 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -104,6 +104,7 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { function unmask(value: string): string { let parts = value.split("*"); + let placeholder = "x"; if (parts.length === 2) { let textCount = parseInt(parts[0], 36); let wordCount = parseInt(parts[1], 36); @@ -111,16 +112,16 @@ function unmask(value: string): string { if (wordCount > 0 && textCount === 0) { value = " "; } else if (wordCount === 0 && textCount > 0) { - value = Array(textCount + 1).join("x"); + value = Array(textCount + 1).join(placeholder); } else if (wordCount > 0 && textCount > 0) { value = ""; let avg = Math.floor(textCount / wordCount); while (value.length < textCount + wordCount) { let gap = Math.min(avg, textCount + wordCount - value.length); - value += Array(gap + 1).join("x") + " "; + value += Array(gap + 1).join(placeholder) + " "; } } else { - value = Array(Math.floor((textCount + 1) / 2) + 1).join("x "); + value = Array(textCount + wordCount + 1).join(placeholder); } } } diff --git a/decode/render.ts b/decode/render.ts index 9cbbbf4f..f5f458c9 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -57,6 +57,7 @@ export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { el.style.minWidth = Math.max(bm.box[2] - 5, 0) + "px"; el.style.maxHeight = bm.box[3] + "px"; el.style.overflow = "hidden"; + el.style.wordBreak = "break-all"; } } } diff --git a/src/data/upload.ts b/src/data/upload.ts index 601e202f..f94fd143 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -20,6 +20,7 @@ function send(data: string): void { if (config.url.length > 0) { let xhr = new XMLHttpRequest(); xhr.open("POST", config.url); + xhr.setRequestHeader("Content-Type", "application/json"); xhr.send(data); } } diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 00698828..2137a7cf 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -149,7 +149,16 @@ function leaf(tag: string, id: number, parentId: number): void { if (id !== null && parentId !== null) { switch (tag) { case "*T": - values[parentId].leaf = true; + // Mark parent as a leaf node only if the text node has valid text + // For nodes with whitespaces and not real text, skip them + let value = values[id].data.value; + for (let i = 0; i < value.length; i++) { + let code = value.charCodeAt(i); + if (!(code === 32 || code === 10 || code === 9 || code === 13)) { + values[parentId].leaf = true; + break; + } + } break; case "IMG": case "svg:svg": From 94e23344617a2d38d5773f02a78bc269ab4729fa Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sat, 31 Aug 2019 08:53:51 -0700 Subject: [PATCH 070/105] Simplifying metrics --- decode/metric.ts | 60 ++++++++++++++++------------------ decode/render.ts | 32 ++++--------------- src/metric/encode.ts | 24 ++++++-------- src/metric/index.ts | 17 ++++------ types/metric.d.ts | 76 +++++++++++++++++++++----------------------- 5 files changed, 88 insertions(+), 121 deletions(-) diff --git a/decode/metric.ts b/decode/metric.ts index 27e538c0..2cb52010 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -1,28 +1,26 @@ -import { Token } from "../types/data"; +import { Event, Token } from "../types/data"; import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metric"; -let map: IMetricMap = {}; +let metricMap: IMetricMap = {}; -map[Metric.Nodes] = { name: "Node Count", unit: ""}; -map[Metric.Bytes] = { name: "Byte Count", unit: "KB"}; -map[Metric.Mutations] = { name: "Mutation Count", unit: ""}; -map[Metric.Interactions] = { name: "Interaction Count", unit: ""}; -map[Metric.Clicks] = { name: "Click Count", unit: ""}; -map[Metric.ScriptErrors] = { name: "Script Errors", unit: ""}; -map[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; -map[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; -map[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; -map[Metric.BoxModelTime] = { name: "Box Model Time", unit: "ms"}; -map[Metric.WireupTime] = { name: "Wireup Delay", unit: "s"}; -map[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; -map[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px", value: "max"}; -map[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px", value: "max"}; -map[Metric.DocumentWidth] = { name: "Document Width", unit: "px", value: "max"}; -map[Metric.DocumentHeight] = { name: "Document Height", unit: "px", value: "max"}; -map[Metric.ClickEvent] = { name: "Click Event", unit: ""}; -map[Metric.InteractionEvent] = { name: "Interaction Event", unit: ""}; +metricMap[Metric.Nodes] = { name: "Node Count", unit: ""}; +metricMap[Metric.Bytes] = { name: "Byte Count", unit: "KB"}; +metricMap[Metric.Mutations] = { name: "Mutation Count", unit: ""}; +metricMap[Metric.Interactions] = { name: "Interaction Count", unit: ""}; +metricMap[Metric.Clicks] = { name: "Click Count", unit: ""}; +metricMap[Metric.ScriptErrors] = { name: "Script Errors", unit: ""}; +metricMap[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; +metricMap[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; +metricMap[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; +metricMap[Metric.BoxModelTime] = { name: "Box Model Time", unit: "ms"}; +metricMap[Metric.WireupTime] = { name: "Wireup Delay", unit: "s"}; +metricMap[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; +metricMap[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px"}; +metricMap[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; +metricMap[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; +metricMap[Metric.DocumentHeight] = { name: "Document Height", unit: "px"}; -let metrics: IDecodedMetric = { counters: {}, measures: {}, events: [], marks: [], map }; +let metrics: IDecodedMetric = { counters: {}, measures: {}, events: [], marks: [] }; export default function(tokens: Token[]): IDecodedMetric { let i = 0; @@ -37,22 +35,18 @@ export default function(tokens: Token[]): IDecodedMetric { // Parse metrics switch (metricType) { case MetricType.Counter: - metrics.counters[tokens[i++] as Metric] = tokens[i++] as number; + let counter = metricMap[tokens[i++] as Metric]; + metrics.counters[counter.name] = { value: tokens[i++] as number, unit: counter.unit }; break; - case MetricType.Summary: - metrics.measures[tokens[i++] as Metric] = { - sum: tokens[i++] as number, - min: tokens[i++] as number, - max: tokens[i++] as number, - sumsquared: tokens[i++] as number, - count: tokens[i++] as number, - }; + case MetricType.Measure: + let measure = metricMap[tokens[i++] as Metric]; + metrics.measures[measure.name] = { value: tokens[i++] as number, unit: measure.unit }; break; - case MetricType.Events: - metrics.events.push({ metric: tokens[i++] as Metric, time: tokens[i++] as number, duration: tokens[i++] as number }); + case MetricType.Event: + metrics.events.push({ event: tokens[i++] as Event, time: tokens[i++] as number, duration: tokens[i++] as number }); break; case MetricType.Marks: - metrics.marks.push({ name: tokens[i++] as string, time: tokens[i++] as number }); + metrics.marks.push({ key: tokens[i++] as string, value: tokens[i++] as string, time: tokens[i++] as number }); break; } } diff --git a/decode/render.ts b/decode/render.ts index f5f458c9..1afe58f7 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,6 +1,6 @@ import { IResize, IScroll, Scroll } from "../types/interaction"; import { IBoxModel, IDecodedNode } from "../types/layout"; -import { IDecodedMetric, IMetricMapValue } from "../types/metric"; +import { IDecodedMetric } from "../types/metric"; let nodes = {}; let svgns: string = "http://www.w3.org/2000/svg"; @@ -12,24 +12,11 @@ export function reset(): void { export function metrics(data: IDecodedMetric, header: HTMLElement): void { let html = []; - // Counters - for (let metric in data.counters) { - if (data.counters[metric]) { - let map = data.map[metric]; - let v = value(data.counters[metric], map.unit); - html.push(metricBox(v, data.map[metric])); - } - } - - // Summary - for (let metric in data.measures) { - if (data.measures[metric]) { - let m = data.measures[metric]; - let map = data.map[metric]; - let unit = map.unit; - let v = value(map.value ? m[map.value] : m.sum, unit); - let metadata = map.value === "max" ? `#${m.count} Min: ${value(m.min, unit)}` : `#${m.count} Max: ${value(m.max, unit)}`; - html.push(metricBox(v, data.map[metric], metadata)); + let entries = {...data.counters, ...data.measures}; + for (let metric in entries) { + if (entries[metric]) { + let m = entries[metric]; + html.push(`

  • ${value(m.value, m.unit)}${m.unit}

    ${metric}
    • `); } } @@ -44,18 +31,13 @@ function value(input: number, unit: string): number { } } -function metricBox(metric: number, map: IMetricMapValue, metadata: string = null): string { - metadata = metadata || ""; - return `

  • ${metric}${map.unit}
    ${metadata}

    ${map.name}
    • `; -} - export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { for (let bm of data) { let el = element(bm.id) as HTMLElement; if (el) { el.style.maxWidth = bm.box[2] + "px"; el.style.minWidth = Math.max(bm.box[2] - 5, 0) + "px"; - el.style.maxHeight = bm.box[3] + "px"; + el.style.height = bm.box[3] + "px"; el.style.overflow = "hidden"; el.style.wordBreak = "break-all"; } diff --git a/src/metric/encode.ts b/src/metric/encode.ts index 799f261c..90a5ea57 100644 --- a/src/metric/encode.ts +++ b/src/metric/encode.ts @@ -19,28 +19,23 @@ export default function(): Token[] { } // Encode summary metrics - output.push(MetricType.Summary); - let summaries = metrics.measures; - for (let metric in summaries) { - if (summaries[metric]) { + output.push(MetricType.Measure); + let measures = metrics.measures; + for (let metric in measures) { + if (measures[metric]) { let m = num(metric); if (updates.indexOf(m) >= 0) { - let h = summaries[metric]; output.push(m); - output.push(h.sum); - output.push(h.min); - output.push(h.max); - output.push(h.sumsquared); - output.push(h.count); + output.push(measures[metric]); } } } - // Encode semantic events - if (metrics.events.length > 0) { output.push(MetricType.Events); } + // Encode events summary + if (metrics.events.length > 0) { output.push(MetricType.Event); } let events = metrics.events; for (let event of events) { - output.push(event.metric); + output.push(event.event); output.push(event.time); output.push(event.duration); } @@ -49,7 +44,8 @@ export default function(): Token[] { if (metrics.marks.length > 0) { output.push(MetricType.Marks); } let marks = metrics.marks; for (let mark of marks) { - output.push(mark.name); + output.push(mark.key); + output.push(mark.value); output.push(mark.time); } diff --git a/src/metric/index.ts b/src/metric/index.ts index 513c5d7e..2e7a9d94 100644 --- a/src/metric/index.ts +++ b/src/metric/index.ts @@ -1,3 +1,4 @@ +import { Event } from "@clarity-types/data"; import { IMetric, Metric } from "@clarity-types/metric"; import time from "@src/core/time"; @@ -19,21 +20,17 @@ export function counter(metric: Metric, increment: number = 1): void { } export function measure(metric: Metric, value: number): void { - if (!(metric in metrics.measures)) { metrics.measures[metric] = { sum: 0, min: null, max: null, sumsquared: 0, count: 0 }; } - metrics.measures[metric].sum += value; - metrics.measures[metric].min = metrics.measures[metric].min !== null ? Math.min(metrics.measures[metric].min, value) : value; - metrics.measures[metric].max = metrics.measures[metric].max !== null ? Math.max(metrics.measures[metric].max, value) : value; - metrics.measures[metric].sumsquared += (value * value); - metrics.measures[metric].count++; + if (!(metric in metrics.measures)) { metrics.measures[metric] = 0; } + metrics.measures[metric] = Math.max(value, metrics.measures[metric]); track(metric); } -export function event(metric: Metric, begin: number, duration: number = 0): void { - metrics.events.push({ metric, time: begin, duration }); +export function event(evt: Event, begin: number, duration: number = 0): void { + metrics.events.push({ event: evt, time: begin, duration }); } -export function mark(name: string): void { - metrics.marks.push({ name, time: time() }); +export function mark(key: string, value: string): void { + metrics.marks.push({ key, value, time: time() }); } function track(metric: Metric): void { diff --git a/types/metric.d.ts b/types/metric.d.ts index ea8e1154..475f583b 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -1,9 +1,10 @@ +import { Event } from "./data"; + export const enum MetricType { Counter = "C", - Timing = "T", - Summary = "S", - Events = "E", - Marks = "M" + Measure = "M", + Event = "E", + Marks = "K" } export const enum Metric { @@ -20,58 +21,55 @@ export const enum Metric { BoxModelTime, WireupTime, ActiveTime, - /* Summary */ + /* Measures */ ViewportWidth, ViewportHeight, DocumentWidth, - DocumentHeight, - /* Semantic Events */ - ClickEvent, - InteractionEvent + DocumentHeight } -export interface IMetricMap { - [key: number]: IMetricMapValue; +export interface IMetric { + counters: IMetricValue; + measures: IMetricValue; + events: IEventMetric[]; + marks: IMarkMetric[]; } -export interface IMetricMapValue { - name: string; - unit: string; - value?: string; +export interface IMetricValue { + [key: number]: number; } -export interface IMetric { - counters: ICounter; - measures: IMeasure; - events: ISemanticEvent[]; - marks: IMark[]; +export interface IEventMetric { + event: Event; + time: number; + duration: number; } -export interface IDecodedMetric extends IMetric { - map: IMetricMap; +export interface IMarkMetric { + key: string; + value: string; + time: number; } -export interface ICounter { - [key: number]: number; +export interface IMetricMap { + [key: number]: IMetricMapValue; } -export interface IMeasure { - [key: number]: { - sum: number; - min: number; - max: number; - count: number; - sumsquared: number; - }; +export interface IMetricMapValue { + name: string; + unit: string; } -export interface ISemanticEvent { - metric: number; - time: number; - duration: number; +export interface IDecodedMetric { + counters: IDecodedMetricValue; + measures: IDecodedMetricValue; + events: IEventMetric[]; + marks: IMarkMetric[]; } -export interface IMark { - name: string; - time: number; +export interface IDecodedMetricValue { + [key: string]: { + value: number; + unit: string; + }; } From 18e625845cb709f2d6ffac7f0abe4926f14fa120 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 1 Sep 2019 07:13:32 -0700 Subject: [PATCH 071/105] Deleting unused code --- src/core/task.ts | 4 ++-- src/layout/encode.ts | 25 +------------------------ types/core.d.ts | 10 +--------- 3 files changed, 4 insertions(+), 35 deletions(-) diff --git a/src/core/task.ts b/src/core/task.ts index 20d403c0..37e51a2f 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,10 +1,10 @@ -import { IAsyncTask, ITaskTiming, TaskCallback, TaskFunction } from "@clarity-types/core"; +import { IAsyncTask, ITaskTracker, TaskCallback, TaskFunction } from "@clarity-types/core"; import {Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as metrics from "@src/metric"; -let tracker: ITaskTiming = {}; +let tracker: ITaskTracker = {}; let threshold = config.longtask; let queue: IAsyncTask[] = []; let active: IAsyncTask = null; diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 69ba7578..d4969bc8 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -10,8 +10,6 @@ import * as boxmodel from "./boxmodel"; import {doc} from "./document"; import * as dom from "./dom"; -window["HASH"] = hash; - export default async function(type: Event): Promise { let tokens: Token[] = [time(), type]; let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; @@ -36,9 +34,8 @@ export default async function(type: Event): Promise { for (let value of values) { if (task.longtask(timer)) { await task.idle(timer); } let metadata = []; - let layouts = []; let data: INodeData = value.data; - let keys = ["tag", "layout", "attributes", "value"]; + let keys = ["tag", "attributes", "value"]; for (let key of keys) { if (data[key]) { switch (key) { @@ -56,14 +53,6 @@ export default async function(type: Event): Promise { } } break; - case "layout": - if (data[key].length > 0) { - let boxes = layout(data[key]); - for (let box of boxes) { - layouts.push(box); - } - } - break; case "value": let parent = dom.getNode(value.parent); let parentTag = dom.get(parent) ? dom.get(parent).data.tag : null; @@ -80,10 +69,6 @@ export default async function(type: Event): Promise { let index: number = typeof token === "string" ? tokens.indexOf(token) : -1; tokens.push(index >= 0 && token.length > index.toString().length ? [index] : token); } - // Add layout boxes - for (let entry of layouts) { - tokens.push(entry); - } } return tokens; } @@ -132,11 +117,3 @@ function mask(value: string): string { } return `${textCount.toString(36)}*${wordCount.toString(36)}`; } - -function layout(l: number[]): string[] { - let output = []; - for (let i = 0; i < l.length; i = i + 4) { - output.push([l[i + 0].toString(36), l[i + 1].toString(36), l[i + 2].toString(36), l[i + 3].toString(36)].join("*")); - } - return output; -} diff --git a/types/core.d.ts b/types/core.d.ts index 32c148f9..a0bf6a2e 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -26,15 +26,7 @@ export interface IConfig { upload?: (data: string) => void; } -// Task -export const enum Task { - Discover, - Mutation, - Wireup, - Active -} - -export interface ITaskTiming { +export interface ITaskTracker { [key: number]: number; } From 58e91185f5c83f2b6abb3afcefa57ccacb0f433b Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 2 Sep 2019 16:24:22 -0700 Subject: [PATCH 072/105] Adding support for checksum and some refactoring --- decode/layout.ts | 41 ++++++++++++++++++++-- src/core/config.ts | 3 +- src/core/task.ts | 2 +- src/layout/boxmodel.ts | 10 +++--- src/layout/discover.ts | 3 +- src/layout/dom.ts | 79 ++++++++++++++++++++++++++++++++---------- src/layout/encode.ts | 25 ++++++++++--- src/layout/mutation.ts | 3 +- types/core.d.ts | 3 +- types/data.d.ts | 1 + types/layout.d.ts | 13 ++++++- 11 files changed, 147 insertions(+), 36 deletions(-) diff --git a/decode/layout.ts b/decode/layout.ts index e91cc485..32badf6a 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -1,13 +1,16 @@ import { resolve } from "../src/data/token"; import { Event, IDecodedEvent, Token } from "../types/data"; -import { IBoxModel, IDecodedNode, IDocumentSize } from "../types/layout"; +import { IAttributes, IBoxModel, IChecksum, IDecodedNode, IDocumentSize, } from "../types/layout"; let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; +let selectorMap = {}; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; let event = tokens[1] as Event; let decoded: IDecodedEvent = {time, event, data: []}; + selectorMap = {}; + switch (event) { case Event.Document: let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; @@ -19,6 +22,16 @@ export default function(tokens: Token[]): IDecodedEvent { decoded.data.push(boxmodel); } return decoded; + case Event.Checksum: + let reference = 0; + for (let i = 2; i < tokens.length; i += 2) { + let id = (tokens[i] as number) + reference; + let token = tokens[i + 1]; + let checksum: IChecksum = { id, checksum: typeof(token) === "object" ? tokens[token[0]] : token }; + decoded.data.push(checksum); + reference = id; + } + return decoded; case Event.Discover: case Event.Mutation: let lastType = null; @@ -69,16 +82,22 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { id: node[0], parent: tagIndex > 1 ? node[1] : null, next: tagIndex > 2 ? node[2] : null, - tag: node[tagIndex] + tag: node[tagIndex], + selector: "", }; let hasAttribute = false; let attributes = {}; let value = null; + let path = output.parent in selectorMap ? `${selectorMap[output.parent]}>` : null; + for (let i = tagIndex + 1; i < node.length; i++) { let token = node[i] as string; let keyIndex = token.indexOf("="); + let lastChar = token[token.length - 1]; if (i === (node.length - 1) && output.tag === "STYLE") { value = token; + } else if (lastChar === ">" && keyIndex === -1) { + path = token; } else if (output.tag !== "*T" && keyIndex > 0) { hasAttribute = true; let k = token.substr(0, keyIndex); @@ -96,12 +115,30 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { } } + output.selector = selector(output.id, path, output.tag, attributes); if (hasAttribute) { output.attributes = attributes; } if (value) { output.value = value; } return output; } +function selector(id: number, path: string, tag: string, attributes: IAttributes): string { + switch (tag) { + case "STYLE": + case "TITLE": + case "LINK": + case "META": + case "*T": + return ""; + default: + let s = path && path.length > 0 ? path + tag : tag; + if ("id" in attributes) { s = `${tag}#${attributes["id"]}`; } + if ("class" in attributes) { s += `.${attributes["class"].trim().split(" ").join(".")}`; } + selectorMap[id] = s; + return s; + } +} + function unmask(value: string): string { let parts = value.split("*"); let placeholder = "x"; diff --git a/src/core/config.ts b/src/core/config.ts index 2e0e43cd..b30d9264 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -4,11 +4,12 @@ let config: IConfig = { pageId: null, userId: null, projectId: null, - longtask: 30, + longTask: 30, lookahead: 500, distance: 20, delay: 1000, cssRules: false, + diet: false, tokens: [], url: "", upload: null diff --git a/src/core/task.ts b/src/core/task.ts index 37e51a2f..5b742e25 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -5,7 +5,7 @@ import config from "@src/core/config"; import * as metrics from "@src/metric"; let tracker: ITaskTracker = {}; -let threshold = config.longtask; +let threshold = config.longTask; let queue: IAsyncTask[] = []; let active: IAsyncTask = null; diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index c22a8999..a716a385 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -8,7 +8,7 @@ import encode from "@src/layout/encode"; import * as dom from "./dom"; let bm: {[key: number]: IBoxModel} = {}; -let updates: number[] = []; +let updateMap: number[] = []; let timeout: number = null; export function compute(): void { @@ -46,12 +46,12 @@ async function boxmodel(): Promise { return data; } -export function summarize(): IBoxModel[] { +export function updates(): IBoxModel[] { let summary = []; - for (let id of updates) { + for (let id of updateMap) { summary.push(bm[id]); } - updates = []; + updateMap = []; return summary; } function update(id: number, box: number[]): void { @@ -69,7 +69,7 @@ function update(id: number, box: number[]): void { } if (changed) { - if (updates.indexOf(id) === -1) { updates.push(id); } + if (updateMap.indexOf(id) === -1) { updateMap.push(id); } bm[id] = {id, box}; } } diff --git a/src/layout/discover.ts b/src/layout/discover.ts index 5bb2e566..dbb2b608 100644 --- a/src/layout/discover.ts +++ b/src/layout/discover.ts @@ -1,6 +1,7 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; +import config from "@src/core/config"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; import * as boxmodel from "@src/layout/boxmodel"; @@ -29,7 +30,7 @@ async function discover(): Promise { processNode(node, Source.Discover); node = walker.nextNode(); } - let data = await encode(Event.Discover); + let data = await encode(config.diet ? Event.Checksum : Event.Discover); task.stop(timer); return data; } diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 2137a7cf..5b9105d0 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -9,14 +9,15 @@ let index: number = 1; let nodes: Node[] = []; let values: INodeValue[] = []; -let updates: number[] = []; let changes: INodeChange[][] = []; +let updateMap: number[] = []; +let selectorMap: number[] = []; export function reset(): void { index = 1; nodes = []; values = []; - updates = []; + updateMap = []; changes = []; if (DEVTOOLS_HOOK in window) { window[DEVTOOLS_HOOK] = { get, getNode, history }; } } @@ -35,10 +36,12 @@ export function add(node: Node, data: INodeData, source: Source): void { let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); let masked = true; + let parent = null; if (parentId >= 0 && values[parentId]) { - values[parentId].children.push(id); - masked = values[parentId].masked; + parent = values[parentId]; + parent.children.push(id); + masked = parent.metadata.masked; } if (data.attributes && MASK_ATTRIBUTE in data.attributes) { masked = true; } @@ -51,9 +54,8 @@ export function add(node: Node, data: INodeData, source: Source): void { next: nextId, children: [], data, - active: true, - leaf: false, - masked + selector: selector(id, data, parent ? parent.selector : ""), + metadata: { active: true, leaf: false, masked } }; leaf(data.tag, id, parentId); track(id, source); @@ -85,7 +87,7 @@ export function update(node: Node, data: INodeData, source: Source): void { } } else { // Mark this element as deleted if the parent has been updated to null - value.active = false; + value.metadata.active = false; } // Remove reference to this node from the old parent @@ -115,6 +117,13 @@ export function getNode(id: number): Node { return null; } +export function getValue(id: number): INodeValue { + if (id in values) { + return values[id]; + } + return null; +} + export function get(node: Node): INodeValue { let id = getId(node); return values[id]; @@ -127,24 +136,58 @@ export function has(node: Node): boolean { export function getLeafNodes(): INodeValue[] { let v = []; for (let id in values) { - if (values[id].active && values[id].leaf) { + if (values[id].metadata.active && values[id].metadata.leaf) { v.push(values[id]); } } return v; } -export function summarize(): INodeValue[] { +export function updates(): INodeValue[] { + let output = []; + for (let id of updateMap) { + if (id in values) { + let v = values[id]; + let p = v.parent; + let hasId = "attributes" in v.data && "id" in v.data.attributes; + v.data.path = p === null || p in updateMap || hasId || v.selector.length === 0 ? null : values[p].selector; + output.push(values[id]); + } + } + updateMap = []; + return output; +} + +export function selectors(): INodeValue[] { let v = []; - for (let id of updates) { + for (let id of selectorMap) { if (id in values) { v.push(values[id]); } } - updates = []; + selectorMap = []; return v; } +function selector(id: number, data: INodeData, parent: string): string { + switch (data.tag) { + case "STYLE": + case "TITLE": + case "LINK": + case "META": + case "*T": + return ""; + default: + let value = getValue(id); + let ex = value ? value.selector : null; + let attributes = "attributes" in data ? data.attributes : {}; + let current = "id" in attributes ? `${data.tag}#${attributes.id}` : `${parent}>${data.tag}`; + if ("class" in attributes) { current = `${current}.${attributes.class.trim().split(" ").join(".")}`; } + if (current !== ex && selectorMap.indexOf(id) === -1) { selectorMap.push(id); } + return current; + } +} + function leaf(tag: string, id: number, parentId: number): void { if (id !== null && parentId !== null) { switch (tag) { @@ -155,14 +198,14 @@ function leaf(tag: string, id: number, parentId: number): void { for (let i = 0; i < value.length; i++) { let code = value.charCodeAt(i); if (!(code === 32 || code === 10 || code === 9 || code === 13)) { - values[parentId].leaf = true; + values[parentId].metadata.leaf = true; break; } } break; case "IMG": case "svg:svg": - values[id].leaf = true; + values[id].metadata.leaf = true; break; } } @@ -185,11 +228,11 @@ function track(id: number, source: Source): void { // Keep track of the order in which mutations happened, they may not be sequential // Edge case: If an element is added later on, and pre-discovered element is moved as a child. // In that case, we need to reorder the prediscovered element in the update list to keep visualization consistent. - let uIndex = updates.indexOf(id); + let uIndex = updateMap.indexOf(id); if (uIndex >= 0 && source === Source.ChildListAdd) { - updates.splice(uIndex, 1); - updates.push(id); - } else if (uIndex === -1) { updates.push(id); } + updateMap.splice(uIndex, 1); + updateMap.push(id); + } else if (uIndex === -1) { updateMap.push(id); } if (DEVTOOLS_HOOK in window) { let value = copy([values[id]])[0]; diff --git a/src/layout/encode.ts b/src/layout/encode.ts index d4969bc8..33637656 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -22,20 +22,32 @@ export default async function(type: Event): Promise { metric.measure(Metric.DocumentHeight, d.height); return tokens; case Event.BoxModel: - let bm = boxmodel.summarize(); + let bm = boxmodel.updates(); for (let value of bm) { tokens.push(value.id); tokens.push(value.box); } return tokens; + case Event.Checksum: + let selectors = dom.selectors(); + let reference = 0; + for (let value of selectors) { + if (task.longtask(timer)) { await task.idle(timer); } + let checksum = hash(value.selector); + let pointer = tokens.indexOf(checksum); + tokens.push(value.id - reference); + tokens.push(pointer >= 0 ? [pointer] : checksum); + reference = value.id; + } + return tokens; case Event.Discover: case Event.Mutation: - let values = dom.summarize(); + let values = dom.updates(); for (let value of values) { if (task.longtask(timer)) { await task.idle(timer); } let metadata = []; let data: INodeData = value.data; - let keys = ["tag", "attributes", "value"]; + let keys = ["tag", "path", "attributes", "value"]; for (let key of keys) { if (data[key]) { switch (key) { @@ -46,10 +58,13 @@ export default async function(type: Event): Promise { if (value.next) { tokens.push(value.next); } metadata.push(data[key]); break; + case "path": + metadata.push(`${value.data.path}>`); + break; case "attributes": for (let attr in data[key]) { if (data[key][attr] !== undefined) { - metadata.push(attribute(value.masked, attr, data[key][attr])); + metadata.push(attribute(value.metadata.masked, attr, data[key][attr])); } } break; @@ -57,7 +72,7 @@ export default async function(type: Event): Promise { let parent = dom.getNode(value.parent); let parentTag = dom.get(parent) ? dom.get(parent).data.tag : null; let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; - metadata.push(text(value.masked, tag, data[key])); + metadata.push(text(value.metadata.masked, tag, data[key])); break; } } diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index d3f74c62..4ae06702 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -1,6 +1,7 @@ import { Event, Token } from "@clarity-types/data"; import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; +import config from "@src/core/config"; import * as task from "@src/core/task"; import queue from "@src/data/queue"; import * as boxmodel from "@src/layout/boxmodel"; @@ -96,7 +97,7 @@ async function process(): Promise { break; } } - let data = await encode(Event.Mutation); + let data = await encode(config.diet ? Event.Checksum : Event.Mutation); task.stop(timer); return data; } diff --git a/types/core.d.ts b/types/core.d.ts index a0bf6a2e..e9a6a367 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -16,11 +16,12 @@ export interface IConfig { pageId?: string; userId?: string; projectId?: string; - longtask?: number; + longTask?: number; lookahead?: number; distance?: number; delay?: number; cssRules?: boolean; + diet?: boolean; tokens?: string[]; url?: string; upload?: (data: string) => void; diff --git a/types/data.d.ts b/types/data.d.ts index 4ac0e7d9..65fa4bae 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -8,6 +8,7 @@ export const enum Event { Discover, Mutation, BoxModel, + Checksum, Mouse, Touch, Keyboard, diff --git a/types/layout.d.ts b/types/layout.d.ts index 8010bda9..13ff464c 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -12,6 +12,7 @@ export interface IAttributes { export interface INodeData { tag: string; + path?: string; attributes?: IAttributes; value?: string; } @@ -22,7 +23,11 @@ export interface INodeValue { next: number; children: number[]; data: INodeData; - /* Metadata */ + selector: string; + metadata: INodeMetadata; +} + +export interface INodeMetadata { active: boolean; leaf: boolean; masked: boolean; @@ -39,6 +44,7 @@ export interface IDecodedNode { parent: number; next: number; tag: string; + selector: string; attributes?: IAttributes; value?: string; } @@ -52,3 +58,8 @@ export interface IBoxModel { id: number; box: number[]; } + +export interface IChecksum { + id: number; + checksum: string; +} From 07895ef25fa4e615bb8818cd45de0222398038cc Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 2 Sep 2019 16:50:51 -0700 Subject: [PATCH 073/105] Supporting boxmodel visualization --- decode/clarity.ts | 4 ++++ decode/render.ts | 35 ++++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index dfe8e197..593ab1a0 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -37,6 +37,7 @@ export function json(data: string): IDecodedPayload { case Event.Discover: case Event.Mutation: case Event.BoxModel: + case Event.Checksum: event = layout(entry); break; case Event.Metadata: @@ -69,6 +70,9 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head case Event.Mutation: r.markup(entry.data, iframe); break; + case Event.Checksum: + r.checksum(entry.data, iframe); + break; case Event.BoxModel: r.boxmodel(entry.data, iframe); break; diff --git a/decode/render.ts b/decode/render.ts index 1afe58f7..cfe2cd2c 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,9 +1,10 @@ import { IResize, IScroll, Scroll } from "../types/interaction"; -import { IBoxModel, IDecodedNode } from "../types/layout"; +import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; import { IDecodedMetric } from "../types/metric"; let nodes = {}; let svgns: string = "http://www.w3.org/2000/svg"; +let diet = false; export function reset(): void { nodes = {}; @@ -31,15 +32,35 @@ function value(input: number, unit: string): number { } } +export function checksum(data: IChecksum[], iframe: HTMLIFrameElement): void { + diet = true; +} + export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { + let doc = iframe.contentDocument; for (let bm of data) { let el = element(bm.id) as HTMLElement; - if (el) { - el.style.maxWidth = bm.box[2] + "px"; - el.style.minWidth = Math.max(bm.box[2] - 5, 0) + "px"; - el.style.height = bm.box[3] + "px"; - el.style.overflow = "hidden"; - el.style.wordBreak = "break-all"; + switch (diet) { + case true: + let box = el ? el : doc.createElement("DIV"); + box.style.left = bm.box[0] + "px"; + box.style.top = bm.box[1] + "px"; + box.style.width = bm.box[2] + "px"; + box.style.height = bm.box[3] + "px"; + box.style.position = "absolute"; + box.style.border = "1px solid red"; + doc.body.appendChild(box); + nodes[bm.id] = box; + break; + case false: + if (el) { + el.style.maxWidth = bm.box[2] + "px"; + el.style.minWidth = Math.max(bm.box[2] - 5, 0) + "px"; + el.style.height = bm.box[3] + "px"; + el.style.overflow = "hidden"; + el.style.wordBreak = "break-all"; + } + break; } } } From 6f7513e578337833719daebcbfe9585a84bff9db Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 3 Sep 2019 07:23:01 -0700 Subject: [PATCH 074/105] Adding support for text selection --- decode/clarity.ts | 4 ++++ decode/interaction.ts | 14 ++++++++--- decode/metric.ts | 1 + decode/render.ts | 34 +++++++++++++++----------- src/core/config.ts | 2 +- src/interaction/encode.ts | 10 ++++++++ src/interaction/index.ts | 2 ++ src/interaction/selection.ts | 46 ++++++++++++++++++++++++++++++++++++ src/layout/discover.ts | 2 +- src/layout/mutation.ts | 2 +- types/core.d.ts | 2 +- types/interaction.d.ts | 7 ++++++ types/metric.d.ts | 1 + 13 files changed, 106 insertions(+), 21 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 593ab1a0..9e9b217f 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -32,6 +32,7 @@ export function json(data: string): IDecodedPayload { case Event.Scroll: case Event.Document: case Event.Resize: + case Event.Selection: event = interaction(entry); break; case Event.Discover: @@ -76,6 +77,9 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head case Event.BoxModel: r.boxmodel(entry.data, iframe); break; + case Event.Selection: + r.selection(entry.data[0], iframe); + break; case Event.Resize: r.resize(entry.data[0], iframe); break; diff --git a/decode/interaction.ts b/decode/interaction.ts index b55eff48..da4ed78d 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -1,5 +1,5 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -import { IResize, IScroll, Scroll } from "../types/interaction"; +import { IResize, IScroll, ISelection, Scroll } from "../types/interaction"; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; @@ -7,8 +7,16 @@ export default function(tokens: Token[]): IDecodedEvent { let decoded: IDecodedEvent = {time, event, data: []}; switch (event) { case Event.Resize: - let r: IResize = { width: tokens[2] as number, height: tokens[3] as number }; - decoded.data.push(r); + decoded.data.push({ width: tokens[2] as number, height: tokens[3] as number } as IResize); + break; + case Event.Selection: + decoded.data.push({ + start: tokens[2] as number, + startOffset: tokens[3] as number, + end: tokens[4] as number, + endOffset: tokens[5] as number + } as ISelection); + break; case Event.Scroll: let i = 2; let scrollType = null; diff --git a/decode/metric.ts b/decode/metric.ts index 2cb52010..157c4a4b 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -8,6 +8,7 @@ metricMap[Metric.Bytes] = { name: "Byte Count", unit: "KB"}; metricMap[Metric.Mutations] = { name: "Mutation Count", unit: ""}; metricMap[Metric.Interactions] = { name: "Interaction Count", unit: ""}; metricMap[Metric.Clicks] = { name: "Click Count", unit: ""}; +metricMap[Metric.Selections] = { name: "Selection Count", unit: ""}; metricMap[Metric.ScriptErrors] = { name: "Script Errors", unit: ""}; metricMap[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; metricMap[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; diff --git a/decode/render.ts b/decode/render.ts index cfe2cd2c..da56313f 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,4 +1,4 @@ -import { IResize, IScroll, Scroll } from "../types/interaction"; +import { IResize, IScroll, ISelection, Scroll } from "../types/interaction"; import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; import { IDecodedMetric } from "../types/metric"; @@ -194,25 +194,31 @@ export function scroll(data: IScroll[], iframe: HTMLIFrameElement): void { } } -export function resize(data: IResize, placeholder: HTMLIFrameElement): void { - placeholder.removeAttribute("style"); +export function resize(data: IResize, iframe: HTMLIFrameElement): void { + iframe.removeAttribute("style"); let margin = 10; let px = "px"; let width = data.width; let height = data.height; - let availableWidth = (placeholder.contentWindow.innerWidth - (2 * margin)); + let availableWidth = (iframe.contentWindow.innerWidth - (2 * margin)); let scaleWidth = Math.min(availableWidth / width, 1); - let scaleHeight = Math.min((placeholder.contentWindow.innerHeight - (16 * margin)) / height, 1); + let scaleHeight = Math.min((iframe.contentWindow.innerHeight - (16 * margin)) / height, 1); let scale = Math.min(scaleWidth, scaleHeight); - placeholder.style.position = "relative"; - placeholder.style.width = width + px; - placeholder.style.height = height + px; - placeholder.style.left = ((availableWidth - (width * scale)) / 2) + px; - placeholder.style.transformOrigin = "0 0 0"; - placeholder.style.transform = "scale(" + scale + ")"; - placeholder.style.border = "1px solid #cccccc"; - placeholder.style.margin = margin + px; - placeholder.style.overflow = "hidden"; + iframe.style.position = "relative"; + iframe.style.width = width + px; + iframe.style.height = height + px; + iframe.style.left = ((availableWidth - (width * scale)) / 2) + px; + iframe.style.transformOrigin = "0 0 0"; + iframe.style.transform = "scale(" + scale + ")"; + iframe.style.border = "1px solid #cccccc"; + iframe.style.margin = margin + px; + iframe.style.overflow = "hidden"; +} + +export function selection(data: ISelection, iframe: HTMLIFrameElement): void { + let doc = iframe.contentDocument; + let s = doc.getSelection(); + s.setBaseAndExtent(element(data.start), data.startOffset, element(data.end), data.endOffset); } function getNode(id: number): HTMLElement { diff --git a/src/core/config.ts b/src/core/config.ts index b30d9264..8cd09ff9 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -9,7 +9,7 @@ let config: IConfig = { distance: 20, delay: 1000, cssRules: false, - diet: false, + thrift: false, tokens: [], url: "", upload: null diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 56e46e63..2e5bc01f 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -6,6 +6,7 @@ import * as metric from "@src/metric"; import * as mouse from "./mouse"; import * as resize from "./resize"; import * as scroll from "./scroll"; +import * as selection from "./selection"; import * as visibility from "./visibility"; export default function(type: Event): Token[] { @@ -49,6 +50,15 @@ export default function(type: Event): Token[] { metric.measure(Metric.ViewportHeight, r.height); resize.reset(); break; + case Event.Selection: + let sl = selection.data; + tokens.push(sl.start); + tokens.push(sl.startOffset); + tokens.push(sl.end); + tokens.push(sl.endOffset); + metric.counter(Metric.Selections); + selection.reset(); + break; case Event.Scroll: let s = scroll.summarize(); let scrollType: Scroll = null; diff --git a/src/interaction/index.ts b/src/interaction/index.ts index c670ebf0..9cd48eaa 100644 --- a/src/interaction/index.ts +++ b/src/interaction/index.ts @@ -1,6 +1,7 @@ import * as mouse from "@src/interaction/mouse"; import * as resize from "@src/interaction/resize"; import * as scroll from "@src/interaction/scroll"; +import * as selection from "@src/interaction/selection"; import * as visibility from "@src/interaction/visibility"; export function start(): void { @@ -8,6 +9,7 @@ export function start(): void { resize.start(); visibility.start(); scroll.start(); + selection.start(); } export function end(): void { diff --git a/src/interaction/selection.ts b/src/interaction/selection.ts index e69de29b..1f0adcb8 100644 --- a/src/interaction/selection.ts +++ b/src/interaction/selection.ts @@ -0,0 +1,46 @@ +import { Event } from "@clarity-types/data"; +import { ISelection } from "@clarity-types/interaction"; +import config from "@src/core/config"; +import { bind } from "@src/core/event"; +import queue from "@src/data/queue"; +import { getId } from "@src/layout/dom"; +import encode from "./encode"; + +export let data: ISelection = null; +let selection: Selection = null; +let timeout: number = null; + +export function start(): void { + reset(); + bind(document, "selectstart", recompute, true); + bind(document, "selectionchange", recompute, true); +} + +function recompute(): void { + let s = document.getSelection(); + + if (selection !== null && data.start !== null && data.start !== getId(s.anchorNode)) { + if (timeout) { clearTimeout(timeout); } + schedule(); + } + + data = { + start: getId(s.anchorNode), + startOffset: s.anchorOffset, + end: getId(s.focusNode), + endOffset: s.focusOffset + }; + selection = s; + + if (timeout) { clearTimeout(timeout); } + timeout = window.setTimeout(schedule, config.lookahead); +} + +function schedule(): void { + queue(encode(Event.Selection)); +} + +export function reset(): void { + selection = null; + data = { start: 0, startOffset: 0, end: 0, endOffset: 0 }; +} diff --git a/src/layout/discover.ts b/src/layout/discover.ts index dbb2b608..499b91c0 100644 --- a/src/layout/discover.ts +++ b/src/layout/discover.ts @@ -30,7 +30,7 @@ async function discover(): Promise { processNode(node, Source.Discover); node = walker.nextNode(); } - let data = await encode(config.diet ? Event.Checksum : Event.Discover); + let data = await encode(config.thrift ? Event.Checksum : Event.Discover); task.stop(timer); return data; } diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index 4ae06702..f42811bc 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -97,7 +97,7 @@ async function process(): Promise { break; } } - let data = await encode(config.diet ? Event.Checksum : Event.Mutation); + let data = await encode(config.thrift ? Event.Checksum : Event.Mutation); task.stop(timer); return data; } diff --git a/types/core.d.ts b/types/core.d.ts index e9a6a367..212b45df 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -21,7 +21,7 @@ export interface IConfig { distance?: number; delay?: number; cssRules?: boolean; - diet?: boolean; + thrift?: boolean; tokens?: string[]; url?: string; upload?: (data: string) => void; diff --git a/types/interaction.d.ts b/types/interaction.d.ts index d6219c24..16cb645f 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -36,3 +36,10 @@ export interface IScroll { export interface IPageVisibility { visible: string; } + +export interface ISelection { + start: number; + startOffset: number; + end: number; + endOffset: number; +} diff --git a/types/metric.d.ts b/types/metric.d.ts index 475f583b..152e6e35 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -14,6 +14,7 @@ export const enum Metric { Mutations, Interactions, Clicks, + Selections, ScriptErrors, ImageErrors, DiscoverTime, From c832984b40cf260d38487539a7176ca2f8a69ea0 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 3 Sep 2019 19:54:31 -0700 Subject: [PATCH 075/105] Adding support for mouse playback --- decode/clarity.ts | 41 ++++++++++++++++------ decode/interaction.ts | 72 +++++++++++++++++++++++++++------------ decode/layout.ts | 10 +++--- decode/metadata.ts | 6 ++-- decode/render.ts | 51 +++++++++++++++++++++------ src/interaction/encode.ts | 2 +- 6 files changed, 131 insertions(+), 51 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 9e9b217f..6c22a445 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -27,29 +27,31 @@ export function json(data: string): IDecodedPayload { payloads.push(payload); for (let entry of encoded) { - let event: IDecodedEvent; + let events: IDecodedEvent[]; switch (entry[1]) { case Event.Scroll: case Event.Document: case Event.Resize: case Event.Selection: - event = interaction(entry); + case Event.Mouse: + events = interaction(entry); break; case Event.Discover: case Event.Mutation: case Event.BoxModel: case Event.Checksum: - event = layout(entry); + events = layout(entry); break; case Event.Metadata: - event = metadata(entry); + events = metadata(entry); break; default: - event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; + events = [{time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}]; break; } - decoded.events.push(event); + decoded.events.push(...events); } + decoded.events.sort(sort); return decoded; } @@ -63,9 +65,15 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head // Render metrics r.metrics(decoded.metrics, header); - // Render events - let events = decoded.events; + // Replay events + replay(decoded.events, iframe); +} + +export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement): Promise { + let start = events[0].time; for (let entry of events) { + if (entry.time - start > 16) { start = await wait(entry.time); } + switch (entry.event) { case Event.Discover: case Event.Mutation: @@ -77,11 +85,14 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head case Event.BoxModel: r.boxmodel(entry.data, iframe); break; + case Event.Mouse: + r.mouse(entry.data, iframe); + break; case Event.Selection: - r.selection(entry.data[0], iframe); + r.selection(entry.data, iframe); break; case Event.Resize: - r.resize(entry.data[0], iframe); + r.resize(entry.data, iframe); break; case Event.Scroll: r.scroll(entry.data, iframe); @@ -89,3 +100,13 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head } } } + +async function wait(timestamp: number): Promise { + return new Promise((resolve: FrameRequestCallback): void => { + setTimeout(resolve, 100, timestamp); + }); +} + +function sort(a: IDecodedEvent, b: IDecodedEvent): number { + return a.time - b.time; +} diff --git a/decode/interaction.ts b/decode/interaction.ts index da4ed78d..36f98cb0 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -1,39 +1,67 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -import { IResize, IScroll, ISelection, Scroll } from "../types/interaction"; +import { IMouse, IResize, IScroll, ISelection, Mouse, Scroll } from "../types/interaction"; -export default function(tokens: Token[]): IDecodedEvent { +export default function(tokens: Token[]): IDecodedEvent[] { let time = tokens[0] as number; let event = tokens[1] as Event; - let decoded: IDecodedEvent = {time, event, data: []}; + let events: IDecodedEvent[] = []; switch (event) { + case Event.Mouse: + let m = 2; + let mouseType = null; + let mouseTarget = null; + let mouseTime = 0; + while (m < tokens.length) { + if (typeof(tokens[m]) === "string") { + mouseType = tokens[m++] as Mouse; + mouseTarget = tokens[m++] as number; + continue; + } + mouseTime += tokens[m++] as number; + let mouseData: IMouse = { + type: mouseType, + target: mouseTarget, + time: mouseTime, + x: tokens[m++] as number, + y: tokens[m++] as number, + buttons: mouseType === Mouse.Click ? tokens[m++] as number : 0 + }; + events.push({ time: mouseTime, event, data: mouseData}); + } + break; case Event.Resize: - decoded.data.push({ width: tokens[2] as number, height: tokens[3] as number } as IResize); + let resizeData: IResize = { + width: tokens[2] as number, + height: tokens[3] as number + }; + events.push({ time, event, data: resizeData }); break; case Event.Selection: - decoded.data.push({ + let selectionData: ISelection = { start: tokens[2] as number, startOffset: tokens[3] as number, end: tokens[4] as number, endOffset: tokens[5] as number - } as ISelection); + }; + events.push({ time, event, data: selectionData }); break; case Event.Scroll: - let i = 2; - let scrollType = null; - let target = null; - let t = 0; - while (i < tokens.length) { - if (typeof(tokens[i]) === "string") { - scrollType = tokens[i++] as Scroll; - target = tokens[i++] as number; - continue; - } - t += tokens[i++] as number; - let v = tokens[i++] as number; - let s: IScroll = { type: scrollType, target, time: t, value: v }; - decoded.data.push(s); + let s = 2; + let scrollType = null; + let target = null; + let scrollTime = 0; + while (s < tokens.length) { + if (typeof(tokens[s]) === "string") { + scrollType = tokens[s++] as Scroll; + target = tokens[s++] as number; + continue; } - break; + scrollTime += tokens[s++] as number; + let scrollValue = tokens[s++] as number; + let scrollData: IScroll = { type: scrollType, target, time: scrollTime, value: scrollValue }; + events.push({ time: scrollTime, event, data: scrollData }); + } + break; } - return decoded; + return events; } diff --git a/decode/layout.ts b/decode/layout.ts index 32badf6a..b11852b1 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -5,7 +5,7 @@ import { IAttributes, IBoxModel, IChecksum, IDecodedNode, IDocumentSize, } from let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; let selectorMap = {}; -export default function(tokens: Token[]): IDecodedEvent { +export default function(tokens: Token[]): IDecodedEvent[] { let time = tokens[0] as number; let event = tokens[1] as Event; let decoded: IDecodedEvent = {time, event, data: []}; @@ -15,13 +15,13 @@ export default function(tokens: Token[]): IDecodedEvent { case Event.Document: let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; decoded.data.push(d); - return decoded; + return [decoded]; case Event.BoxModel: for (let i = 2; i < tokens.length; i += 2) { let boxmodel: IBoxModel = { id: tokens[i] as number, box: tokens[i + 1] as number[] }; decoded.data.push(boxmodel); } - return decoded; + return [decoded]; case Event.Checksum: let reference = 0; for (let i = 2; i < tokens.length; i += 2) { @@ -31,7 +31,7 @@ export default function(tokens: Token[]): IDecodedEvent { decoded.data.push(checksum); reference = id; } - return decoded; + return [decoded]; case Event.Discover: case Event.Mutation: let lastType = null; @@ -73,7 +73,7 @@ export default function(tokens: Token[]): IDecodedEvent { } // Process last node decoded.data.push(process(node, tagIndex)); - return decoded; + return [decoded]; } } diff --git a/decode/metadata.ts b/decode/metadata.ts index 20f07af7..4cdacce0 100644 --- a/decode/metadata.ts +++ b/decode/metadata.ts @@ -1,7 +1,7 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -export default function(tokens: Token[]): IDecodedEvent { - return { +export default function(tokens: Token[]): IDecodedEvent[] { + return [{ time: tokens[0] as number, event: tokens[1] as Event, data: { @@ -14,5 +14,5 @@ export default function(tokens: Token[]): IDecodedEvent { title: tokens[8] as string, referrer: tokens[9] as string } - }; + }]; } diff --git a/decode/render.ts b/decode/render.ts index da56313f..b307a0ea 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,10 +1,15 @@ -import { IResize, IScroll, ISelection, Scroll } from "../types/interaction"; +import { IMouse, IResize, IScroll, ISelection, Mouse, Scroll } from "../types/interaction"; import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; import { IDecodedMetric } from "../types/metric"; let nodes = {}; let svgns: string = "http://www.w3.org/2000/svg"; -let diet = false; +let thrift = false; + +// tslint:disable-next-line: max-line-length +let pointerIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S912AAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6JQAAgIMAAPn/AACA6AAAdTAAAOpgAAA6lwAAF2+XqZnUAAACaUlEQVR4nGL8f58BHYgAsT8Q2wGxBBAzQcX/AfFrID4CxOuA+BWKLoX/YAoggBjRDHQD4ngglgRiPgyrIOAzEL8E4lVQg1EMBAggFiSFYUAcA8RSOAyCAV4oTgViTiBeiiwJEEAw71gRaRgyEAXiKCB2RBYECCCQgcIMEG+SYhgMiANxEhDzwwQAAghkoAMQK5NhGAwoALE1jAMQQCADQU7mpMBAZqijwAAggEAGqgAxOwUGskHNAAOAAAIZyEtIh4INg3bfHHD6xAUEYAyAAAIZ+IuQgU9fMLCXdzDIzV3JIIhDyQ8YAyCAQAaCUv8/fAZysDP8+/OXgTG7jkFhwRoMQ0F6n8M4AAEEMvAKsg34wM9fDEwgQ1dtRSQTIPgNxFdhHIAAAhm4AYg/EmMgCHz7zsCUVMaguHob3FCQYzbD5AECCGTgJSDeCbWJKPD1GwNzSjmD4tZ9DFxgvQr/b8PkAAIIlvVWA/FuUgz99IWBOTyXQcE+nOEOsjhAACGXNnJAHAnE9kAshqyIV5vB4Ms3cALGBkAlj9////9PgTgAAcSEJPEIiDuBeBYQP2CAhOt3BsLJCpSfNzAyMpqDOAABhF4ewh3FAMmf2kAsyqnBUPDjJ8HcdBvoSjWAAGIEEgTUMTAAbf/AwICSVGCgD4hPgJQA8WegWdsBAogFiyJC4C0QgxI3KLj4gIasRpYECCAGkAsJYSAAuRDEAKUEQwZIzgDxvwCxCrJagAAi1kAQAYpFESh/BlQMhJuR1QIEELEGlgOxHBLflAGSh0Gc60DMBpMDCCCiDMRhyXoGSJUaDgpPmDhAgAEAN5Ugk0bMYNIAAAAASUVORK5CYII="; +// tslint:disable-next-line: max-line-length +let clickIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S912AAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6JQAAgIMAAPn/AACA6AAAdTAAAOpgAAA6lwAAF2+XqZnUAAADvklEQVR4nGL8//8/AzJgZGQQAVL+QGwHxBJAzASV+gfEr4H4CBCvA2p7xYAFAAQQI7KBQMPcgFQ8EEsCMR82DUDwGYhfAvEqoNZ16JIAAQQ3EGhYGJCKAWIpHAahA5BrlwC1L0UWBAggJqhhViQaBgKiQBwF1OuILAgQQExAAWEGiDeRDJsEFOM1AFplzMCgrcnAcIoTh6HiQJwENIMfJgAQQCAXOgCxMkLNaqBkvgIDg8Z3BoaC5wwMz9kYGKIVGRg+MjFgB0C1DNYwDkAAgRSBnIzkgnUCDAyswIBdf4+Bof8ZA0PHYwaGO0D5/Tw4DGSGOgoMAAIIZKAKELMj5D8AFXD+BUbyXwhf9Scw5QAt+MmIw0A2qBlgABBAIAN5cSgkBQjAGAABxALEv1BdiA4YobgFmDZPcjMwZLyB+JILmNAl/0AV/YCpBgggkIGg9MTNgMgRWAAL0MuvWRkYJgNzznRgzP4AqmUHGpj/AhgFnxgYlD4yMKiBVQIEEMiQK8g2YAJQ2JUAY/vFZQaGmfeBvgO6qhUYUU5AQ7qASc1Tg4FB35+RkRGUXRkAAgjkwg0MDMedGBh2AW3m+QtxCQuSgZxAl9h+gbAtvzEwJAN9VAXMxzc+Qlwa+JSBoRQUZL1AvBEggECB4wdMJqsYGH6zQ8IKlBWlgOF6/SowpoGGvQKaDgoqKSDxCWjAA2Cs6gF99AXIfgLEGsuA+kAJuwqYjRkBAgjk5S5gQQL00vHJDAwnLgFz1G9gPCElEbE/EMNAAGSBHjR4eIBiGtuBjKVQV4ABQACBDASG5t/dDAwWPQwMZkDbJlyApMG/UEN38UBc9hvIPwCMPFDy/AmM6UnXgN6eDywcniKHOEAAgQw8BsRBQGezASU7gfm9jYGh8hxQIzD2GIDZ7yYwjXwHuuYPMIHfApr2HxghbxcBY/giA4PmE6TUAgYAAQRilALxASCeBYwpb2A4bGBkTNnLAMmf2sDYBWbN73cZGDiAZeBxYBlp1w00CBg5DBlIXpUH4mcgBkAAMUDLw2So5CQQHxkDQRwQCzFAEn8oAyRVg/J+IVQM5ChgvmfYDFIPEEDIGudCDQ1HM9CFAZI9gckJXC2AggmUfwOh8mpQfZUgPkAAIWsEaToOxF+B2ATdpbgwEKRCDbQB8QECCF1SB4hBkXEeiPmJNBCUbEDZlwfEBwgg5CwBUnAFGDHpQCYw+TBsAbI3MUBqO2xFFyj9CELDdRlQLzg3AQQQLlujoAH9H6oRG/4HlT8ExCowvQABBgBOKHD8+UgEvgAAAABJRU5ErkJggg=="; export function reset(): void { nodes = {}; @@ -33,14 +38,14 @@ function value(input: number, unit: string): number { } export function checksum(data: IChecksum[], iframe: HTMLIFrameElement): void { - diet = true; + thrift = true; } export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; for (let bm of data) { let el = element(bm.id) as HTMLElement; - switch (diet) { + switch (thrift) { case true: let box = el ? el : doc.createElement("DIV"); box.style.left = bm.box[0] + "px"; @@ -186,12 +191,10 @@ function setAttributes(node: HTMLElement, attributes: object): void { } } -export function scroll(data: IScroll[], iframe: HTMLIFrameElement): void { - for (let d of data) { - let target = getNode(d.target); - if (target && d.type === Scroll.X) { target.scrollTo(d.value, target.scrollTop); } - if (target && d.type === Scroll.Y) { target.scrollTo(target.scrollLeft, d.value); } - } +export function scroll(data: IScroll, iframe: HTMLIFrameElement): void { + let target = getNode(data.target); + if (target && data.type === Scroll.X) { target.scrollTo(data.value, target.scrollTop); } + if (target && data.type === Scroll.Y) { target.scrollTo(target.scrollLeft, data.value); } } export function resize(data: IResize, iframe: HTMLIFrameElement): void { @@ -221,6 +224,34 @@ export function selection(data: ISelection, iframe: HTMLIFrameElement): void { s.setBaseAndExtent(element(data.start), data.startOffset, element(data.end), data.endOffset); } +export function mouse(data: IMouse, iframe: HTMLIFrameElement): void { + let doc = iframe.contentDocument; + let pointer = doc.getElementById("clarity-pointer"); + let pointerWidth = 20; + let pointerHeight = 24; + + if (pointer === null) { + pointer = doc.createElement("DIV"); + pointer.id = "clarity-pointer"; + pointer.style.position = "absolute"; + pointer.style.zIndex = "1000"; + pointer.style.width = pointerWidth + "px"; + pointer.style.height = pointerHeight + "px"; + doc.body.appendChild(pointer); + } + + pointer.style.left = (data.x - 8) + "px"; + pointer.style.top = (data.y - 8) + "px"; + switch (data.type) { + case Mouse.Click: + pointer.style.background = `url(${clickIcon}) no-repeat left center`; + break; + default: + pointer.style.background = `url(${pointerIcon}) no-repeat left center`; + break; + } +} + function getNode(id: number): HTMLElement { return id in nodes ? nodes[id] : null; } diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 2e5bc01f..d85dea3f 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -36,7 +36,7 @@ export default function(type: Event): Token[] { tokens.push(entry.time - timestamp); tokens.push(entry.x); tokens.push(entry.y); - tokens.push(entry.buttons); + if (mouseType === Mouse.Click) { tokens.push(entry.buttons); } timestamp = entry.time; } From 0e6754c81af09f91a8cfb5def141d3bcb9f727b7 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 4 Sep 2019 06:37:31 -0700 Subject: [PATCH 076/105] Refactoring code to mask as a core function --- src/core/mask.ts | 13 +++++++++++++ src/layout/encode.ts | 15 +-------------- 2 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 src/core/mask.ts diff --git a/src/core/mask.ts b/src/core/mask.ts new file mode 100644 index 00000000..a6b46dcb --- /dev/null +++ b/src/core/mask.ts @@ -0,0 +1,13 @@ +export default function(value: string): string { + let wasWhiteSpace = false; + let textCount = 0; + let wordCount = 0; + for (let i = 0; i < value.length; i++) { + let code = value.charCodeAt(i); + let isWhiteSpace = (code === 32 || code === 10 || code === 9 || code === 13); + textCount += isWhiteSpace ? 0 : 1; + wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; + wasWhiteSpace = isWhiteSpace; + } + return `${textCount.toString(36)}*${wordCount.toString(36)}`; +} diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 33637656..604abe5c 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -1,6 +1,7 @@ import {Event, Token} from "@clarity-types/data"; import {INodeData} from "@clarity-types/layout"; import {Metric} from "@clarity-types/metric"; +import mask from "@src/core/mask"; import * as task from "@src/core/task"; import time from "@src/core/time"; import hash from "@src/data/hash"; @@ -118,17 +119,3 @@ function text(masked: boolean, tag: string, value: string): string { return masked ? mask(value) : value; } } - -function mask(value: string): string { - let wasWhiteSpace = false; - let textCount = 0; - let wordCount = 0; - for (let i = 0; i < value.length; i++) { - let code = value.charCodeAt(i); - let isWhiteSpace = (code === 32 || code === 10 || code === 9 || code === 13); - textCount += isWhiteSpace ? 0 : 1; - wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; - wasWhiteSpace = isWhiteSpace; - } - return `${textCount.toString(36)}*${wordCount.toString(36)}`; -} From a68813f9f2a596d7836160243a6d13f961d3aa44 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 4 Sep 2019 08:03:56 -0700 Subject: [PATCH 077/105] Major refactoring to remove event groupings --- decode/clarity.ts | 30 ++++++++++---- decode/interaction.ts | 69 ++++++++----------------------- decode/layout.ts | 10 ++--- decode/metadata.ts | 6 +-- decode/render.ts | 12 +++--- src/core/config.ts | 1 + src/interaction/encode.ts | 70 ++++++++++--------------------- src/interaction/mouse.ts | 77 ++++++++++++++++------------------- src/interaction/resize.ts | 3 +- src/interaction/scroll.ts | 58 ++++++-------------------- src/interaction/selection.ts | 9 +--- src/interaction/visibility.ts | 3 +- types/core.d.ts | 1 + types/data.d.ts | 8 +++- types/interaction.d.ts | 24 ++--------- 15 files changed, 140 insertions(+), 241 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 6c22a445..21ee7d52 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -27,29 +27,35 @@ export function json(data: string): IDecodedPayload { payloads.push(payload); for (let entry of encoded) { - let events: IDecodedEvent[]; + let event: IDecodedEvent; switch (entry[1]) { case Event.Scroll: case Event.Document: case Event.Resize: case Event.Selection: - case Event.Mouse: - events = interaction(entry); + case Event.MouseDown: + case Event.MouseUp: + case Event.MouseMove: + case Event.MouseWheel: + case Event.Click: + case Event.DoubleClick: + case Event.RightClick: + event = interaction(entry); break; case Event.Discover: case Event.Mutation: case Event.BoxModel: case Event.Checksum: - events = layout(entry); + event = layout(entry); break; case Event.Metadata: - events = metadata(entry); + event = metadata(entry); break; default: - events = [{time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}]; + event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; break; } - decoded.events.push(...events); + decoded.events.push(event); } decoded.events.sort(sort); return decoded; @@ -85,8 +91,14 @@ export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement) case Event.BoxModel: r.boxmodel(entry.data, iframe); break; - case Event.Mouse: - r.mouse(entry.data, iframe); + case Event.MouseDown: + case Event.MouseUp: + case Event.MouseMove: + case Event.MouseWheel: + case Event.Click: + case Event.DoubleClick: + case Event.RightClick: + r.mouse(entry.event, entry.data, iframe); break; case Event.Selection: r.selection(entry.data, iframe); diff --git a/decode/interaction.ts b/decode/interaction.ts index 36f98cb0..541a3a96 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -1,41 +1,22 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -import { IMouse, IResize, IScroll, ISelection, Mouse, Scroll } from "../types/interaction"; +import { IMouse, IResize, IScroll, ISelection } from "../types/interaction"; -export default function(tokens: Token[]): IDecodedEvent[] { +export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; let event = tokens[1] as Event; - let events: IDecodedEvent[] = []; switch (event) { - case Event.Mouse: - let m = 2; - let mouseType = null; - let mouseTarget = null; - let mouseTime = 0; - while (m < tokens.length) { - if (typeof(tokens[m]) === "string") { - mouseType = tokens[m++] as Mouse; - mouseTarget = tokens[m++] as number; - continue; - } - mouseTime += tokens[m++] as number; - let mouseData: IMouse = { - type: mouseType, - target: mouseTarget, - time: mouseTime, - x: tokens[m++] as number, - y: tokens[m++] as number, - buttons: mouseType === Mouse.Click ? tokens[m++] as number : 0 - }; - events.push({ time: mouseTime, event, data: mouseData}); - } - break; + case Event.MouseDown: + case Event.MouseUp: + case Event.MouseMove: + case Event.MouseWheel: + case Event.Click: + case Event.DoubleClick: + case Event.RightClick: + let mouseData: IMouse = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; + return { time, event, data: mouseData }; case Event.Resize: - let resizeData: IResize = { - width: tokens[2] as number, - height: tokens[3] as number - }; - events.push({ time, event, data: resizeData }); - break; + let resizeData: IResize = { width: tokens[2] as number, height: tokens[3] as number }; + return { time, event, data: resizeData }; case Event.Selection: let selectionData: ISelection = { start: tokens[2] as number, @@ -43,25 +24,11 @@ export default function(tokens: Token[]): IDecodedEvent[] { end: tokens[4] as number, endOffset: tokens[5] as number }; - events.push({ time, event, data: selectionData }); - break; + return { time, event, data: selectionData }; case Event.Scroll: - let s = 2; - let scrollType = null; - let target = null; - let scrollTime = 0; - while (s < tokens.length) { - if (typeof(tokens[s]) === "string") { - scrollType = tokens[s++] as Scroll; - target = tokens[s++] as number; - continue; - } - scrollTime += tokens[s++] as number; - let scrollValue = tokens[s++] as number; - let scrollData: IScroll = { type: scrollType, target, time: scrollTime, value: scrollValue }; - events.push({ time: scrollTime, event, data: scrollData }); - } - break; + let scrollData: IScroll = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; + return { time, event, data: scrollData }; + default: + return { time, event, data: tokens.slice(2) }; } - return events; } diff --git a/decode/layout.ts b/decode/layout.ts index b11852b1..32badf6a 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -5,7 +5,7 @@ import { IAttributes, IBoxModel, IChecksum, IDecodedNode, IDocumentSize, } from let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; let selectorMap = {}; -export default function(tokens: Token[]): IDecodedEvent[] { +export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; let event = tokens[1] as Event; let decoded: IDecodedEvent = {time, event, data: []}; @@ -15,13 +15,13 @@ export default function(tokens: Token[]): IDecodedEvent[] { case Event.Document: let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; decoded.data.push(d); - return [decoded]; + return decoded; case Event.BoxModel: for (let i = 2; i < tokens.length; i += 2) { let boxmodel: IBoxModel = { id: tokens[i] as number, box: tokens[i + 1] as number[] }; decoded.data.push(boxmodel); } - return [decoded]; + return decoded; case Event.Checksum: let reference = 0; for (let i = 2; i < tokens.length; i += 2) { @@ -31,7 +31,7 @@ export default function(tokens: Token[]): IDecodedEvent[] { decoded.data.push(checksum); reference = id; } - return [decoded]; + return decoded; case Event.Discover: case Event.Mutation: let lastType = null; @@ -73,7 +73,7 @@ export default function(tokens: Token[]): IDecodedEvent[] { } // Process last node decoded.data.push(process(node, tagIndex)); - return [decoded]; + return decoded; } } diff --git a/decode/metadata.ts b/decode/metadata.ts index 4cdacce0..20f07af7 100644 --- a/decode/metadata.ts +++ b/decode/metadata.ts @@ -1,7 +1,7 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -export default function(tokens: Token[]): IDecodedEvent[] { - return [{ +export default function(tokens: Token[]): IDecodedEvent { + return { time: tokens[0] as number, event: tokens[1] as Event, data: { @@ -14,5 +14,5 @@ export default function(tokens: Token[]): IDecodedEvent[] { title: tokens[8] as string, referrer: tokens[9] as string } - }]; + }; } diff --git a/decode/render.ts b/decode/render.ts index b307a0ea..1a79d0c1 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,4 +1,5 @@ -import { IMouse, IResize, IScroll, ISelection, Mouse, Scroll } from "../types/interaction"; +import { Event } from "../types/data"; +import { IMouse, IResize, IScroll, ISelection } from "../types/interaction"; import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; import { IDecodedMetric } from "../types/metric"; @@ -193,8 +194,7 @@ function setAttributes(node: HTMLElement, attributes: object): void { export function scroll(data: IScroll, iframe: HTMLIFrameElement): void { let target = getNode(data.target); - if (target && data.type === Scroll.X) { target.scrollTo(data.value, target.scrollTop); } - if (target && data.type === Scroll.Y) { target.scrollTo(target.scrollLeft, data.value); } + target.scrollTo(data.x, data.y); } export function resize(data: IResize, iframe: HTMLIFrameElement): void { @@ -224,7 +224,7 @@ export function selection(data: ISelection, iframe: HTMLIFrameElement): void { s.setBaseAndExtent(element(data.start), data.startOffset, element(data.end), data.endOffset); } -export function mouse(data: IMouse, iframe: HTMLIFrameElement): void { +export function mouse(event: Event, data: IMouse, iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; let pointer = doc.getElementById("clarity-pointer"); let pointerWidth = 20; @@ -242,8 +242,8 @@ export function mouse(data: IMouse, iframe: HTMLIFrameElement): void { pointer.style.left = (data.x - 8) + "px"; pointer.style.top = (data.y - 8) + "px"; - switch (data.type) { - case Mouse.Click: + switch (event) { + case Event.Click: pointer.style.background = `url(${clickIcon}) no-repeat left center`; break; default: diff --git a/src/core/config.ts b/src/core/config.ts index 8cd09ff9..27e0af2c 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -7,6 +7,7 @@ let config: IConfig = { longTask: 30, lookahead: 500, distance: 20, + interval: 25, delay: 1000, cssRules: false, thrift: false, diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index d85dea3f..3ad30082 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -1,7 +1,7 @@ import {Event, Token} from "@clarity-types/data"; -import {Mouse, Scroll} from "@clarity-types/interaction"; import {Metric} from "@clarity-types/metric"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import * as metric from "@src/metric"; import * as mouse from "./mouse"; import * as resize from "./resize"; @@ -9,36 +9,24 @@ import * as scroll from "./scroll"; import * as selection from "./selection"; import * as visibility from "./visibility"; -export default function(type: Event): Token[] { +export default function(type: Event): void { let tokens: Token[] = [time(), type]; - let timestamp: number = null; switch (type) { - case Event.Mouse: - let m = mouse.summarize(); - timestamp = null; - let mouseType: Mouse = null; - let mouseTarget: number = null; + case Event.MouseDown: + case Event.MouseUp: + case Event.MouseMove: + case Event.MouseWheel: + case Event.Click: + case Event.DoubleClick: + case Event.RightClick: + let m = mouse.data[type]; for (let i = 0; i < m.length; i++) { let entry = m[i]; - - if (i === 0) { - timestamp = entry.time; - tokens[0] = timestamp; - } - - if (mouseType !== entry.type || mouseTarget !== entry.target) { - tokens.push(entry.type); - tokens.push(entry.target); - mouseType = entry.type; - mouseTarget = entry.target; - } - - tokens.push(entry.time - timestamp); + tokens = [entry.time, type]; + tokens.push(entry.target); tokens.push(entry.x); tokens.push(entry.y); - if (mouseType === Mouse.Click) { tokens.push(entry.buttons); } - - timestamp = entry.time; + queue(tokens); } mouse.reset(); break; @@ -46,6 +34,7 @@ export default function(type: Event): Token[] { let r = resize.data; tokens.push(r.width); tokens.push(r.height); + queue(tokens); metric.measure(Metric.ViewportWidth, r.width); metric.measure(Metric.ViewportHeight, r.height); resize.reset(); @@ -56,42 +45,27 @@ export default function(type: Event): Token[] { tokens.push(sl.startOffset); tokens.push(sl.end); tokens.push(sl.endOffset); + queue(tokens); metric.counter(Metric.Selections); selection.reset(); break; case Event.Scroll: - let s = scroll.summarize(); - let scrollType: Scroll = null; - let scrollTarget: number = null; - timestamp = null; + let s = scroll.data; for (let i = 0; i < s.length; i++) { let entry = s[i]; - - if (i === 0) { - timestamp = entry.time; - tokens[0] = timestamp; - } - - if (scrollType !== entry.type || scrollTarget !== entry.target) { - tokens.push(entry.type); - tokens.push(entry.target); - scrollType = entry.type; - scrollTarget = entry.target; - } - - tokens.push(entry.time - timestamp); - tokens.push(entry.value); - - timestamp = entry.time; + tokens = [entry.time, type]; + tokens.push(entry.target); + tokens.push(entry.x); + tokens.push(entry.y); + queue(tokens); } scroll.reset(); break; case Event.Visibility: let v = visibility.data; tokens.push(v.visible); + queue(tokens); visibility.reset(); break; } - - return tokens; } diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index df859391..6af72d62 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -1,65 +1,58 @@ import { Event } from "@clarity-types/data"; -import { IMouse, Mouse } from "@clarity-types/interaction"; +import { IMouse } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; -import queue from "@src/data/queue"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -let data: IMouse[] = []; +export let data: { [key: number]: IMouse[] } = []; let timeout: number = null; export function start(): void { - bind(document, "mousedown", handler.bind(this, Mouse.Down)); - bind(document, "mouseup", handler.bind(this, Mouse.Up)); - bind(document, "mousemove", handler.bind(this, Mouse.Move)); - bind(document, "mousewheel", handler.bind(this, Mouse.Wheel)); - bind(document, "dblclick", handler.bind(this, Mouse.DoubleClick)); - bind(document, "click", handler.bind(this, Mouse.Click)); + reset(); + bind(document, "mousedown", handler.bind(this, Event.MouseDown)); + bind(document, "mouseup", handler.bind(this, Event.MouseUp)); + bind(document, "mousemove", handler.bind(this, Event.MouseMove)); + bind(document, "mousewheel", handler.bind(this, Event.MouseWheel)); + bind(document, "dblclick", handler.bind(this, Event.DoubleClick)); + bind(document, "click", handler.bind(this, Event.Click)); } -function handler(type: Mouse, evt: MouseEvent): void { +function handler(event: Event, evt: MouseEvent): void { let de = document.documentElement; - data.push({ - type, - target: evt.target ? getId(evt.target as Node) : null, - time: time(), - x: "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null), - y: "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null), - buttons: evt.buttons - }); - if (timeout) { clearTimeout(timeout); } - timeout = window.setTimeout(schedule, config.lookahead); -} + let x = "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null); + let y = "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null); + let current = {target: evt.target ? getId(evt.target as Node) : null, x, y, time: time()}; + switch (event) { + case Event.MouseMove: + case Event.MouseWheel: + let length = data[event].length; + let last = length > 1 ? data[event][length - 2] : null; + if (last && similar(last, current)) { data[event].pop(); } + data[event].push(current); -function schedule(): void { - queue(encode(Event.Mouse)); + if (timeout) { clearTimeout(timeout); } + timeout = window.setTimeout(encode, config.lookahead, event); + break; + case Event.Click: + event = evt.buttons === 2 || evt.button === 2 ? Event.RightClick : event; + default: + data[event].push(current); + encode(event); + break; + } } export function reset(): void { - data = []; -} - -export function summarize(): IMouse[] { - let summary: IMouse[] = []; - let index = 0; - let last = null; - for (let entry of data) { - let isFirst = index === 0; - if (isFirst - || index === data.length - 1 - || checkDistance(last, entry)) { - summary.push(entry); - } - index++; - last = entry; + data = {}; + for (let event of [Event.MouseDown, Event.MouseUp, Event.MouseMove, Event.DoubleClick, Event.Click]) { + data[event] = []; } - return summary; } -function checkDistance(last: IMouse, current: IMouse): boolean { +function similar(last: IMouse, current: IMouse): boolean { let dx = last.x - current.x; let dy = last.y - current.y; - return (dx * dx + dy * dy > config.distance * config.distance); + return (dx * dx + dy * dy < config.distance * config.distance) && (current.time - last.time < config.interval); } diff --git a/src/interaction/resize.ts b/src/interaction/resize.ts index fe8304b1..63323842 100644 --- a/src/interaction/resize.ts +++ b/src/interaction/resize.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; import { IResize } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; -import queue from "@src/data/queue"; import encode from "./encode"; export let data: IResize; @@ -16,7 +15,7 @@ function recompute(): void { width: "innerWidth" in window ? window.innerWidth : document.documentElement.clientWidth, height: "innerHeight" in window ? window.innerHeight : document.documentElement.clientHeight }; - queue(encode(Event.Resize)); + encode(Event.Resize); } export function reset(): void { diff --git a/src/interaction/scroll.ts b/src/interaction/scroll.ts index cceeb9b2..fc5eebe1 100644 --- a/src/interaction/scroll.ts +++ b/src/interaction/scroll.ts @@ -1,16 +1,12 @@ import { Event } from "@clarity-types/data"; -import { IScroll, Scroll } from "@clarity-types/interaction"; +import { IScroll } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; -import queue from "@src/data/queue"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -let lastX = {}; -let lastY = {}; -let dataX: IScroll[] = []; -let dataY: IScroll[] = []; +export let data: IScroll[] = []; let timeout: number = null; export function start(): void { @@ -19,54 +15,26 @@ export function start(): void { } function recompute(event: UIEvent = null): void { - let t = time(); let eventTarget = event ? (event.target === document ? document.documentElement : event.target) : document.documentElement; - let target = getId(eventTarget as Node); let x = (eventTarget as HTMLElement).scrollLeft; let y = (eventTarget as HTMLElement).scrollTop; + let current: IScroll = {target: getId(eventTarget as Node), x, y, time: time()}; - if (x !== lastX[target]) { - dataX.push({ target, type: Scroll.X, time: t, value: x }); - lastX[target] = x; - } - - if (y !== lastY[target]) { - dataY.push({ target, type: Scroll.Y, time: t, value: y }); - lastY[target] = y; - } + let length = data.length; + let last = length > 1 ? data[length - 2] : null; + if (last && similar(last, current)) { data.pop(); } + data.push(current); if (timeout) { clearTimeout(timeout); } - timeout = window.setTimeout(schedule, config.lookahead); -} - -function schedule(): void { - queue(encode(Event.Scroll)); + timeout = window.setTimeout(encode, config.lookahead, Event.Scroll); } export function reset(): void { - dataX = []; - dataY = []; - lastX = {}; - lastY = {}; -} - -export function summarize(): IScroll[] { - let summary: IScroll[] = []; - let last = null; - let data = dataX.concat(dataY); - for (let i = 0; i < data.length; i++) { - let entry = data[i]; - let isFirst = i === 0; - let isLast = i === data.length - 1; - if (isFirst || isLast || checkDistance(last, entry)) { - summary.push(entry); - last = entry; - } - } - return summary; + data = []; } -function checkDistance(last: IScroll, current: IScroll): boolean { - let d = last.value - current.value; - return (d > config.distance); +function similar(last: IScroll, current: IScroll): boolean { + let dx = last.x - current.x; + let dy = last.y - current.y; + return (dx * dx + dy * dy < config.distance * config.distance) && (current.time - last.time < config.interval); } diff --git a/src/interaction/selection.ts b/src/interaction/selection.ts index 1f0adcb8..2183fbd3 100644 --- a/src/interaction/selection.ts +++ b/src/interaction/selection.ts @@ -2,7 +2,6 @@ import { Event } from "@clarity-types/data"; import { ISelection } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; -import queue from "@src/data/queue"; import { getId } from "@src/layout/dom"; import encode from "./encode"; @@ -21,7 +20,7 @@ function recompute(): void { if (selection !== null && data.start !== null && data.start !== getId(s.anchorNode)) { if (timeout) { clearTimeout(timeout); } - schedule(); + encode(Event.Selection); } data = { @@ -33,11 +32,7 @@ function recompute(): void { selection = s; if (timeout) { clearTimeout(timeout); } - timeout = window.setTimeout(schedule, config.lookahead); -} - -function schedule(): void { - queue(encode(Event.Selection)); + timeout = window.setTimeout(encode, config.lookahead, Event.Selection); } export function reset(): void { diff --git a/src/interaction/visibility.ts b/src/interaction/visibility.ts index 3df4dec8..aa92f5b1 100644 --- a/src/interaction/visibility.ts +++ b/src/interaction/visibility.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; import { IPageVisibility } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; -import queue from "@src/data/queue"; import encode from "./encode"; export let data: IPageVisibility; @@ -15,7 +14,7 @@ export function start(): void { function recompute(): void { data = { visible: "visibilityState" in document ? document.visibilityState : "default" }; - queue(encode(Event.Visibility)); + encode(Event.Visibility); } export function reset(): void { diff --git a/types/core.d.ts b/types/core.d.ts index 212b45df..7fb1e17c 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -19,6 +19,7 @@ export interface IConfig { longTask?: number; lookahead?: number; distance?: number; + interval?: number; delay?: number; cssRules?: boolean; thrift?: boolean; diff --git a/types/data.d.ts b/types/data.d.ts index 65fa4bae..9ed42a6e 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -9,7 +9,13 @@ export const enum Event { Mutation, BoxModel, Checksum, - Mouse, + Click, + MouseMove, + MouseDown, + MouseUp, + MouseWheel, + DoubleClick, + RightClick, Touch, Keyboard, Selection, diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 16cb645f..9c7590c8 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -1,19 +1,8 @@ -export const enum Mouse { - Down = "D", - Up = "U", - Move = "M", - Wheel = "W", - DoubleClick = "B", - Click = "C" -} - export interface IMouse { - type: Mouse; target: number; - time: number; x: number; y: number; - buttons: number; + time?: number; } export interface IResize { @@ -21,16 +10,11 @@ export interface IResize { height: number; } -export const enum Scroll { - X = "X", - Y = "Y" -} - export interface IScroll { - type: Scroll; target: number; - time: number; - value: number; + x: number; + y: number; + time?: number; } export interface IPageVisibility { From 2579848ce9c23373f7623ef836a797ff25dae2cd Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 4 Sep 2019 19:36:38 -0700 Subject: [PATCH 078/105] Simplifying task manager and layout module --- decode/render.ts | 2 +- src/core/task.ts | 17 ++++++++++------- src/layout/boxmodel.ts | 14 ++++---------- src/layout/discover.ts | 19 +++++++------------ src/layout/document.ts | 7 ++----- src/layout/encode.ts | 15 ++++++++++----- src/layout/mutation.ts | 19 +++++++------------ types/core.d.ts | 6 +++--- 8 files changed, 44 insertions(+), 55 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index 1a79d0c1..f8b4d04b 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -194,7 +194,7 @@ function setAttributes(node: HTMLElement, attributes: object): void { export function scroll(data: IScroll, iframe: HTMLIFrameElement): void { let target = getNode(data.target); - target.scrollTo(data.x, data.y); + if (target) { target.scrollTo(data.x, data.y); } } export function resize(data: IResize, iframe: HTMLIFrameElement): void { diff --git a/src/core/task.ts b/src/core/task.ts index 5b742e25..c11457d8 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,5 +1,4 @@ -import { IAsyncTask, ITaskTracker, TaskCallback, TaskFunction } from "@clarity-types/core"; -import {Token } from "@clarity-types/data"; +import { IAsyncTask, ITaskTracker, TaskFunction, TaskResolve } from "@clarity-types/core"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as metrics from "@src/metric"; @@ -9,7 +8,7 @@ let threshold = config.longTask; let queue: IAsyncTask[] = []; let active: IAsyncTask = null; -export async function schedule(task: TaskFunction, callback: TaskCallback): Promise { +export async function schedule(task: TaskFunction): Promise { // If this task is already scheduled, skip it for (let q of queue) { if (q.task === task) { @@ -17,17 +16,21 @@ export async function schedule(task: TaskFunction, callback: TaskCallback): Prom } } - // Otherwise, add thit to the queue - queue.push({task, callback}); + let promise = new Promise((resolve: TaskResolve): void => { + queue.push({task, resolve}); + }); + if (active === null) { run(); } + + return promise; } function run(): void { let entry = queue.shift(); if (entry) { active = entry; - entry.task().then((data: Token[]) => { - entry.callback(data); + entry.task().then(() => { + entry.resolve(); active = null; run(); }); diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index a716a385..e5d6f010 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -1,9 +1,8 @@ -import { Event, Token } from "@clarity-types/data"; +import { Event } from "@clarity-types/data"; import { IBoxModel } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as task from "@src/core/task"; -import queue from "@src/data/queue"; import encode from "@src/layout/encode"; import * as dom from "./dom"; @@ -17,14 +16,10 @@ export function compute(): void { } function schedule(): void { - task.schedule(boxmodel, done); + task.schedule(boxmodel); } -function done(data: Token[]): void { - queue(data); -} - -async function boxmodel(): Promise { +async function boxmodel(): Promise { let timer = Metric.BoxModelTime; task.start(timer); let values = dom.getLeafNodes(); @@ -41,9 +36,8 @@ async function boxmodel(): Promise { update(value.id, getLayout(x, y, dom.getNode(value.id) as Element)); } - let data = await encode(Event.BoxModel); + await encode(Event.BoxModel); task.stop(timer); - return data; } export function updates(): IBoxModel[] { diff --git a/src/layout/discover.ts b/src/layout/discover.ts index 499b91c0..a0ab5983 100644 --- a/src/layout/discover.ts +++ b/src/layout/discover.ts @@ -1,9 +1,8 @@ -import { Event, Token } from "@clarity-types/data"; +import { Event } from "@clarity-types/data"; import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as task from "@src/core/task"; -import queue from "@src/data/queue"; import * as boxmodel from "@src/layout/boxmodel"; import * as doc from "@src/layout/document"; import encode from "@src/layout/encode"; @@ -11,16 +10,13 @@ import encode from "@src/layout/encode"; import processNode from "./node"; export function start(): void { - task.schedule(discover, done); + task.schedule(discover).then(() => { + doc.compute(); + boxmodel.compute(); + }); } -function done(data: Token[]): void { - doc.compute(); - boxmodel.compute(); - queue(data); -} - -async function discover(): Promise { +async function discover(): Promise { let timer = Metric.DiscoverTime; task.start(timer); let walker = document.createTreeWalker(document, NodeFilter.SHOW_ALL, null, false); @@ -30,7 +26,6 @@ async function discover(): Promise { processNode(node, Source.Discover); node = walker.nextNode(); } - let data = await encode(config.thrift ? Event.Checksum : Event.Discover); + await encode(config.thrift ? Event.Checksum : Event.Discover); task.stop(timer); - return data; } diff --git a/src/layout/document.ts b/src/layout/document.ts index 9572189d..619866ed 100644 --- a/src/layout/document.ts +++ b/src/layout/document.ts @@ -1,6 +1,5 @@ -import { Event, Token } from "@clarity-types/data"; +import { Event } from "@clarity-types/data"; import { IDocumentSize } from "@clarity-types/layout"; -import queue from "@src/data/queue"; import encode from "./encode"; export let doc: IDocumentSize; @@ -22,7 +21,5 @@ export function compute(): void { height: documentHeight }; - encode(Event.Document).then((data: Token[]) => { - queue(data); - }); + encode(Event.Document); } diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 604abe5c..7f67cf4a 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -5,13 +5,14 @@ import mask from "@src/core/mask"; import * as task from "@src/core/task"; import time from "@src/core/time"; import hash from "@src/data/hash"; +import queue from "@src/data/queue"; import {check} from "@src/data/token"; import * as metric from "@src/metric"; import * as boxmodel from "./boxmodel"; import {doc} from "./document"; import * as dom from "./dom"; -export default async function(type: Event): Promise { +export default async function(type: Event): Promise { let tokens: Token[] = [time(), type]; let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; switch (type) { @@ -21,14 +22,16 @@ export default async function(type: Event): Promise { tokens.push(d.height); metric.measure(Metric.DocumentWidth, d.width); metric.measure(Metric.DocumentHeight, d.height); - return tokens; + queue(tokens); + break; case Event.BoxModel: let bm = boxmodel.updates(); for (let value of bm) { tokens.push(value.id); tokens.push(value.box); } - return tokens; + queue(tokens); + break; case Event.Checksum: let selectors = dom.selectors(); let reference = 0; @@ -40,7 +43,8 @@ export default async function(type: Event): Promise { tokens.push(pointer >= 0 ? [pointer] : checksum); reference = value.id; } - return tokens; + queue(tokens); + break; case Event.Discover: case Event.Mutation: let values = dom.updates(); @@ -86,7 +90,8 @@ export default async function(type: Event): Promise { tokens.push(index >= 0 && token.length > index.toString().length ? [index] : token); } } - return tokens; + queue(tokens); + break; } } diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index f42811bc..74c57bc2 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -1,9 +1,8 @@ -import { Event, Token } from "@clarity-types/data"; +import { Event } from "@clarity-types/data"; import { Source } from "@clarity-types/layout"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as task from "@src/core/task"; -import queue from "@src/data/queue"; import * as boxmodel from "@src/layout/boxmodel"; import * as doc from "@src/layout/document"; import encode from "@src/layout/encode"; @@ -49,16 +48,13 @@ export function end(): void { function handle(m: MutationRecord[]): void { // Queue up mutation records for asynchronous processing for (let i = 0; i < m.length; i++) { mutations.push(m[i]); } - task.schedule(process, done); + task.schedule(process).then(() => { + doc.compute(); + boxmodel.compute(); + }); } -function done(data: Token[]): void { - doc.compute(); - boxmodel.compute(); - queue(data); -} - -async function process(): Promise { +async function process(): Promise { let timer = Metric.MutationTime; task.start(timer); while (mutations.length > 0) { @@ -97,9 +93,8 @@ async function process(): Promise { break; } } - let data = await encode(config.thrift ? Event.Checksum : Event.Mutation); + await encode(config.thrift ? Event.Checksum : Event.Mutation); task.stop(timer); - return data; } function generate(target: Node, type: MutationRecordType): void { diff --git a/types/core.d.ts b/types/core.d.ts index 7fb1e17c..a8b982e8 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -1,7 +1,7 @@ import { IPayload, Token } from "./data"; -type TaskFunction = () => Promise; -type TaskCallback = (data: Token[]) => void; +type TaskFunction = () => Promise; +type TaskResolve = () => void; export interface IEventBindingPair { target: EventTarget; @@ -34,5 +34,5 @@ export interface ITaskTracker { export interface IAsyncTask { task: TaskFunction; - callback: TaskCallback; + resolve: TaskResolve; } From aac746d6ecc4f1699ecbe9d3d737f6a0055343f6 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 4 Sep 2019 21:38:01 -0700 Subject: [PATCH 079/105] Fixing playback and completing refactoring --- decode/clarity.ts | 2 +- src/data/encode.ts | 19 +++++++++---------- src/data/metadata.ts | 7 +++---- src/diagnostic/encode.ts | 3 +++ src/diagnostic/image.ts | 3 +-- src/diagnostic/script.ts | 3 +-- 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 21ee7d52..0909893a 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -115,7 +115,7 @@ export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement) async function wait(timestamp: number): Promise { return new Promise((resolve: FrameRequestCallback): void => { - setTimeout(resolve, 100, timestamp); + setTimeout(resolve, 10, timestamp); }); } diff --git a/src/data/encode.ts b/src/data/encode.ts index 29fb07ee..b645427b 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -1,24 +1,23 @@ -import {Event, Token} from "@clarity-types/data"; +import {Event, Flush, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import time from "@src/core/time"; import { metadata } from "@src/data/metadata"; import * as metric from "@src/metric"; +import queue from "./queue"; -export default function(envelope: boolean = false): Token[] { +export default function(): void { let t = time(); - let tokens: Token[] = envelope ? [] : [t, Event.Metadata]; + let tokens: Token[] = [t, Event.Metadata]; - if (!envelope) { metric.counter(Metric.WireupTime, t); } + metric.counter(Metric.WireupTime, t); tokens.push(metadata.sequence); tokens.push(metadata.version); tokens.push(metadata.pageId); tokens.push(metadata.userId); tokens.push(metadata.projectId); - if (envelope === false) { - tokens.push(metadata.url); - tokens.push(metadata.title); - tokens.push(metadata.referrer); - } - return tokens; + tokens.push(metadata.url); + tokens.push(metadata.title); + tokens.push(metadata.referrer); + queue(tokens, Flush.None); } diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 75c9271e..4ebd2155 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,9 +1,8 @@ -import { Flush, IMetadata, Token } from "@clarity-types/data"; +import { IMetadata, Token } from "@clarity-types/data"; import config from "@src/core/config"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; -import queue from "@src/data/queue"; export let metadata: IMetadata = null; @@ -19,7 +18,7 @@ export function start(): void { referrer: document.referrer }; - queue(encode(), Flush.None); + encode(); } export function end(): void { @@ -28,7 +27,7 @@ export function end(): void { export function envelope(): Token[] { metadata.sequence++; - return encode(true); + return [metadata.sequence, metadata.version, metadata.pageId, metadata.userId, metadata.projectId]; } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts index 2b13f726..99098a7b 100644 --- a/src/diagnostic/encode.ts +++ b/src/diagnostic/encode.ts @@ -1,6 +1,7 @@ import {Event, Token} from "@clarity-types/data"; import {Metric} from "@clarity-types/metric"; import time from "@src/core/time"; +import queue from "@src/data/queue"; import * as image from "@src/diagnostic/image"; import * as script from "@src/diagnostic/script"; import * as metric from "@src/metric"; @@ -17,6 +18,7 @@ export default function(type: Event): Token[] { tokens.push(e.line); tokens.push(e.column); tokens.push(e.stack); + queue(tokens); metric.counter(Metric.ScriptErrors); } script.reset(); @@ -26,6 +28,7 @@ export default function(type: Event): Token[] { for (let e of images) { tokens.push(e.source); tokens.push(e.target); + queue(tokens); metric.counter(Metric.ImageErrors); } image.reset(); diff --git a/src/diagnostic/image.ts b/src/diagnostic/image.ts index 8a6ccdec..f893bd89 100644 --- a/src/diagnostic/image.ts +++ b/src/diagnostic/image.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; import { IImageError } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; -import queue from "@src/data/queue"; import { getId } from "@src/layout/dom"; import encode from "./encode"; @@ -24,7 +23,7 @@ function handler(error: ErrorEvent): void { }); } - queue(encode(Event.ImageError)); + encode(Event.ImageError); } export function reset(): void { diff --git a/src/diagnostic/script.ts b/src/diagnostic/script.ts index 4031b2e3..ad1f6cf0 100644 --- a/src/diagnostic/script.ts +++ b/src/diagnostic/script.ts @@ -1,7 +1,6 @@ import { Event } from "@clarity-types/data"; import { IScriptError } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; -import queue from "@src/data/queue"; import encode from "./encode"; export let data: IScriptError[] = []; @@ -25,7 +24,7 @@ function handler(error: ErrorEvent): void { source: error["filename"] }); - queue(encode(Event.ScriptError)); + encode(Event.ScriptError); } export function reset(): void { From 5a85a4ac0c6f21fa59db86fae55404d947da8062 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Thu, 5 Sep 2019 05:16:39 -0700 Subject: [PATCH 080/105] Box model replay alignment --- decode/render.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index f8b4d04b..db51ea35 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -60,9 +60,15 @@ export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { break; case false: if (el) { - el.style.maxWidth = bm.box[2] + "px"; - el.style.minWidth = Math.max(bm.box[2] - 5, 0) + "px"; - el.style.height = bm.box[3] + "px"; + let style = getComputedStyle(el, null); + let xPadding = parseInt(style["padding-left"], 10) + parseInt(style["padding-right"], 10); + let xBorder = parseInt(style["border-left"], 10) + parseInt(style["border-right"], 10); + let yPadding = parseInt(style["padding-top"], 10) + parseInt(style["padding-bottom"], 10); + let yBorder = parseInt(style["border-top"], 10) + parseInt(style["border-bottom"], 10); + let width = bm.box[2] - xPadding - xBorder; + el.style.maxWidth = width + "px"; + el.style.minWidth = Math.max(width - 5, 0) + "px"; + el.style.height = (bm.box[3] - yPadding - yBorder) + "px"; el.style.overflow = "hidden"; el.style.wordBreak = "break-all"; } From 867075b82dc44535e4822f482a62a33e2a9bbbad Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 6 Sep 2019 16:43:42 -0700 Subject: [PATCH 081/105] Adding support for change event and lean mode --- decode/clarity.ts | 4 ++++ decode/interaction.ts | 8 +++++++- decode/render.ts | 7 ++++++- src/core/config.ts | 2 +- src/interaction/change.ts | 25 +++++++++++++++++++++++++ src/interaction/encode.ts | 9 +++++++++ src/interaction/keyboard.ts | 0 src/layout/discover.ts | 2 +- src/layout/mutation.ts | 2 +- types/core.d.ts | 2 +- types/data.d.ts | 1 + types/interaction.d.ts | 5 +++++ types/metric.d.ts | 1 + 13 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 src/interaction/change.ts delete mode 100644 src/interaction/keyboard.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 0909893a..4253a8bb 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -33,6 +33,7 @@ export function json(data: string): IDecodedPayload { case Event.Document: case Event.Resize: case Event.Selection: + case Event.Change: case Event.MouseDown: case Event.MouseUp: case Event.MouseMove: @@ -100,6 +101,9 @@ export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement) case Event.RightClick: r.mouse(entry.event, entry.data, iframe); break; + case Event.Change: + r.change(entry.data, iframe); + break; case Event.Selection: r.selection(entry.data, iframe); break; diff --git a/decode/interaction.ts b/decode/interaction.ts index 541a3a96..ac70e9fb 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -1,5 +1,5 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -import { IMouse, IResize, IScroll, ISelection } from "../types/interaction"; +import { IChange, IMouse, IResize, IScroll, ISelection } from "../types/interaction"; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; @@ -17,6 +17,12 @@ export default function(tokens: Token[]): IDecodedEvent { case Event.Resize: let resizeData: IResize = { width: tokens[2] as number, height: tokens[3] as number }; return { time, event, data: resizeData }; + case Event.Change: + let changeData: IChange = { + target: tokens[2] as number, + value: tokens[3] as string + }; + return { time, event, data: changeData }; case Event.Selection: let selectionData: ISelection = { start: tokens[2] as number, diff --git a/decode/render.ts b/decode/render.ts index db51ea35..d5b5e27d 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ import { Event } from "../types/data"; -import { IMouse, IResize, IScroll, ISelection } from "../types/interaction"; +import { IChange, IMouse, IResize, IScroll, ISelection } from "../types/interaction"; import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; import { IDecodedMetric } from "../types/metric"; @@ -224,6 +224,11 @@ export function resize(data: IResize, iframe: HTMLIFrameElement): void { iframe.style.overflow = "hidden"; } +export function change(data: IChange, iframe: HTMLIFrameElement): void { + let el = element(data.target) as HTMLInputElement; + if (el) { el.value = data.value; } +} + export function selection(data: ISelection, iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; let s = doc.getSelection(); diff --git a/src/core/config.ts b/src/core/config.ts index 27e0af2c..e503f2c3 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -10,7 +10,7 @@ let config: IConfig = { interval: 25, delay: 1000, cssRules: false, - thrift: false, + lean: false, tokens: [], url: "", upload: null diff --git a/src/interaction/change.ts b/src/interaction/change.ts new file mode 100644 index 00000000..954ec374 --- /dev/null +++ b/src/interaction/change.ts @@ -0,0 +1,25 @@ +import { Event } from "@clarity-types/data"; +import { IChange } from "@clarity-types/interaction"; +import { bind } from "@src/core/event"; +import mask from "@src/core/mask"; +import { get } from "@src/layout/dom"; +import encode from "./encode"; + +export let data: IChange; + +export function start(): void { + bind(document, "change", recompute, true); +} + +function recompute(evt: UIEvent): void { + let target = evt.target as HTMLInputElement; + let value = get(target); + if (target && value) { + data = { target: value.id, value: value.metadata.masked ? mask(target.value) : target.value }; + encode(Event.Change); + } +} + +export function reset(): void { + data = null; +} diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 3ad30082..1faf174e 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -3,6 +3,7 @@ import {Metric} from "@clarity-types/metric"; import time from "@src/core/time"; import queue from "@src/data/queue"; import * as metric from "@src/metric"; +import * as change from "./change"; import * as mouse from "./mouse"; import * as resize from "./resize"; import * as scroll from "./scroll"; @@ -39,6 +40,14 @@ export default function(type: Event): void { metric.measure(Metric.ViewportHeight, r.height); resize.reset(); break; + case Event.Change: + let ch = change.data; + tokens.push(ch.target); + tokens.push(ch.value); + queue(tokens); + metric.counter(Metric.Changes); + change.reset(); + break; case Event.Selection: let sl = selection.data; tokens.push(sl.start); diff --git a/src/interaction/keyboard.ts b/src/interaction/keyboard.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/layout/discover.ts b/src/layout/discover.ts index a0ab5983..0f174447 100644 --- a/src/layout/discover.ts +++ b/src/layout/discover.ts @@ -26,6 +26,6 @@ async function discover(): Promise { processNode(node, Source.Discover); node = walker.nextNode(); } - await encode(config.thrift ? Event.Checksum : Event.Discover); + await encode(config.lean ? Event.Checksum : Event.Discover); task.stop(timer); } diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index 74c57bc2..788bc417 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -93,7 +93,7 @@ async function process(): Promise { break; } } - await encode(config.thrift ? Event.Checksum : Event.Mutation); + await encode(config.lean ? Event.Checksum : Event.Mutation); task.stop(timer); } diff --git a/types/core.d.ts b/types/core.d.ts index a8b982e8..ce18e010 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -22,7 +22,7 @@ export interface IConfig { interval?: number; delay?: number; cssRules?: boolean; - thrift?: boolean; + lean?: boolean; tokens?: string[]; url?: string; upload?: (data: string) => void; diff --git a/types/data.d.ts b/types/data.d.ts index 9ed42a6e..c1289b4f 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -21,6 +21,7 @@ export const enum Event { Selection, Resize, Scroll, + Change, Document, Visibility, Network, diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 9c7590c8..5cb5daa4 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -27,3 +27,8 @@ export interface ISelection { end: number; endOffset: number; } + +export interface IChange { + target: number; + value: string; +} diff --git a/types/metric.d.ts b/types/metric.d.ts index 152e6e35..85cad916 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -15,6 +15,7 @@ export const enum Metric { Interactions, Clicks, Selections, + Changes, ScriptErrors, ImageErrors, DiscoverTime, From 348e2a27483d2d980457ae45323510b759207d34 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sat, 7 Sep 2019 07:40:13 -0700 Subject: [PATCH 082/105] Rendering fixes to honor box model after mutation --- decode/render.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index d5b5e27d..be1f6a45 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -4,8 +4,9 @@ import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; import { IDecodedMetric } from "../types/metric"; let nodes = {}; +let boxmodels = {}; let svgns: string = "http://www.w3.org/2000/svg"; -let thrift = false; +let lean = false; // tslint:disable-next-line: max-line-length let pointerIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S912AAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6JQAAgIMAAPn/AACA6AAAdTAAAOpgAAA6lwAAF2+XqZnUAAACaUlEQVR4nGL8f58BHYgAsT8Q2wGxBBAzQcX/AfFrID4CxOuA+BWKLoX/YAoggBjRDHQD4ngglgRiPgyrIOAzEL8E4lVQg1EMBAggFiSFYUAcA8RSOAyCAV4oTgViTiBeiiwJEEAw71gRaRgyEAXiKCB2RBYECCCQgcIMEG+SYhgMiANxEhDzwwQAAghkoAMQK5NhGAwoALE1jAMQQCADQU7mpMBAZqijwAAggEAGqgAxOwUGskHNAAOAAAIZyEtIh4INg3bfHHD6xAUEYAyAAAIZ+IuQgU9fMLCXdzDIzV3JIIhDyQ8YAyCAQAaCUv8/fAZysDP8+/OXgTG7jkFhwRoMQ0F6n8M4AAEEMvAKsg34wM9fDEwgQ1dtRSQTIPgNxFdhHIAAAhm4AYg/EmMgCHz7zsCUVMaguHob3FCQYzbD5AECCGTgJSDeCbWJKPD1GwNzSjmD4tZ9DFxgvQr/b8PkAAIIlvVWA/FuUgz99IWBOTyXQcE+nOEOsjhAACGXNnJAHAnE9kAshqyIV5vB4Ms3cALGBkAlj9////9PgTgAAcSEJPEIiDuBeBYQP2CAhOt3BsLJCpSfNzAyMpqDOAABhF4ewh3FAMmf2kAsyqnBUPDjJ8HcdBvoSjWAAGIEEgTUMTAAbf/AwICSVGCgD4hPgJQA8WegWdsBAogFiyJC4C0QgxI3KLj4gIasRpYECCAGkAsJYSAAuRDEAKUEQwZIzgDxvwCxCrJagAAi1kAQAYpFESh/BlQMhJuR1QIEELEGlgOxHBLflAGSh0Gc60DMBpMDCCCiDMRhyXoGSJUaDgpPmDhAgAEAN5Ugk0bMYNIAAAAASUVORK5CYII="; @@ -14,6 +15,7 @@ let clickIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S export function reset(): void { nodes = {}; + boxmodels = {}; } export function metrics(data: IDecodedMetric, header: HTMLElement): void { @@ -39,14 +41,14 @@ function value(input: number, unit: string): number { } export function checksum(data: IChecksum[], iframe: HTMLIFrameElement): void { - thrift = true; + lean = true; } export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; for (let bm of data) { let el = element(bm.id) as HTMLElement; - switch (thrift) { + switch (lean) { case true: let box = el ? el : doc.createElement("DIV"); box.style.left = bm.box[0] + "px"; @@ -70,7 +72,7 @@ export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { el.style.minWidth = Math.max(width - 5, 0) + "px"; el.style.height = (bm.box[3] - yPadding - yBorder) + "px"; el.style.overflow = "hidden"; - el.style.wordBreak = "break-all"; + boxmodels[bm.id] = bm; } break; } @@ -139,6 +141,7 @@ export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { if (!node.attributes) { node.attributes = {}; } node.attributes["data-id"] = `${node.id}`; setAttributes(domElement as HTMLElement, node.attributes); + if (node.id in boxmodels) { boxmodel([boxmodels[node.id]], iframe); } insert(node, parent, domElement, next); break; } From a7e623793e96a0c707baf2390c62b35107c8c9d5 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sat, 7 Sep 2019 21:06:40 -0700 Subject: [PATCH 083/105] Wiring unload handler and some more refactoring --- decode/clarity.ts | 6 +-- decode/envelope.ts | 5 +- decode/metric.ts | 8 ++- decode/{metadata.ts => page.ts} | 0 src/data/encode.ts | 23 +++----- src/data/index.ts | 4 +- src/data/metadata.ts | 26 ++++----- src/data/queue.ts | 32 ----------- src/data/upload.ts | 95 +++++++++++++++++++++++++++------ src/diagnostic/encode.ts | 2 +- src/interaction/encode.ts | 13 ++++- src/interaction/index.ts | 2 + src/interaction/mouse.ts | 2 +- src/interaction/unload.ts | 22 ++++++++ src/interaction/visibility.ts | 4 +- src/layout/encode.ts | 2 +- src/metric/encode.ts | 6 +-- types/core.d.ts | 2 +- types/data.d.ts | 35 +++++++----- types/interaction.d.ts | 6 ++- types/metric.d.ts | 8 ++- 21 files changed, 190 insertions(+), 113 deletions(-) rename decode/{metadata.ts => page.ts} (100%) delete mode 100644 src/data/queue.ts create mode 100644 src/interaction/unload.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 4253a8bb..c8883bff 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -2,8 +2,8 @@ import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types import envelope from "./envelope"; import interaction from "./interaction"; import layout from "./layout"; -import metadata from "./metadata"; import metric from "./metric"; +import page from "./page"; import * as r from "./render"; let pageId: string = null; @@ -49,8 +49,8 @@ export function json(data: string): IDecodedPayload { case Event.Checksum: event = layout(entry); break; - case Event.Metadata: - event = metadata(entry); + case Event.Page: + event = page(entry); break; default: event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; diff --git a/decode/envelope.ts b/decode/envelope.ts index 9579a077..22a8cd90 100644 --- a/decode/envelope.ts +++ b/decode/envelope.ts @@ -1,4 +1,4 @@ -import { IEnvelope, Token } from "../types/data"; +import { IEnvelope, Token, Upload } from "../types/data"; export default function(tokens: Token[]): IEnvelope { return { @@ -6,6 +6,7 @@ export default function(tokens: Token[]): IEnvelope { version: tokens[1] as string, pageId: tokens[2] as string, userId: tokens[3] as string, - projectId: tokens[4] as string + projectId: tokens[4] as string, + upload: tokens[5] as Upload }; } diff --git a/decode/metric.ts b/decode/metric.ts index 157c4a4b..bcb74110 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -4,7 +4,10 @@ import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metric" let metricMap: IMetricMap = {}; metricMap[Metric.Nodes] = { name: "Node Count", unit: ""}; -metricMap[Metric.Bytes] = { name: "Byte Count", unit: "KB"}; +metricMap[Metric.LayoutBytes] = { name: "Layout Bytes", unit: "KB"}; +metricMap[Metric.InteractionBytes] = { name: "Interaction Bytes", unit: "KB"}; +metricMap[Metric.NetworkBytes] = { name: "Network Bytes", unit: "KB"}; +metricMap[Metric.DiagnosticBytes] = { name: "Diagnostic Bytes", unit: "KB"}; metricMap[Metric.Mutations] = { name: "Mutation Count", unit: ""}; metricMap[Metric.Interactions] = { name: "Interaction Count", unit: ""}; metricMap[Metric.Clicks] = { name: "Click Count", unit: ""}; @@ -14,8 +17,9 @@ metricMap[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; metricMap[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; metricMap[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; metricMap[Metric.BoxModelTime] = { name: "Box Model Time", unit: "ms"}; -metricMap[Metric.WireupTime] = { name: "Wireup Delay", unit: "s"}; +metricMap[Metric.LoadTime] = { name: "Load Time", unit: "s"}; metricMap[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; +metricMap[Metric.UnloadTime] = { name: "Unload Time", unit: "s"}; metricMap[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px"}; metricMap[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; metricMap[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; diff --git a/decode/metadata.ts b/decode/page.ts similarity index 100% rename from decode/metadata.ts rename to decode/page.ts diff --git a/src/data/encode.ts b/src/data/encode.ts index b645427b..8f4ed148 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -1,23 +1,16 @@ -import {Event, Flush, Token } from "@clarity-types/data"; +import {Event, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import time from "@src/core/time"; import { metadata } from "@src/data/metadata"; import * as metric from "@src/metric"; -import queue from "./queue"; +import { queue } from "./upload"; export default function(): void { let t = time(); - let tokens: Token[] = [t, Event.Metadata]; - - metric.counter(Metric.WireupTime, t); - - tokens.push(metadata.sequence); - tokens.push(metadata.version); - tokens.push(metadata.pageId); - tokens.push(metadata.userId); - tokens.push(metadata.projectId); - tokens.push(metadata.url); - tokens.push(metadata.title); - tokens.push(metadata.referrer); - queue(tokens, Flush.None); + let tokens: Token[] = [t, Event.Page]; + metric.counter(Metric.LoadTime, t); + tokens.push(metadata.page.url); + tokens.push(metadata.page.title); + tokens.push(metadata.page.referrer); + queue(tokens); } diff --git a/src/data/index.ts b/src/data/index.ts index 11ee459c..b43ed522 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -1,9 +1,11 @@ import * as metadata from "@src/data/metadata"; - +import * as upload from "@src/data/upload"; export function start(): void { + upload.start(); metadata.start(); } export function end(): void { + upload.end(); metadata.end(); } diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 4ebd2155..2469fc63 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,4 +1,4 @@ -import { IMetadata, Token } from "@clarity-types/data"; +import { IEnvelope, IMetadata, IPage, Token, Upload } from "@clarity-types/data"; import config from "@src/core/config"; import version from "@src/core/version"; import encode from "@src/data/encode"; @@ -7,17 +7,12 @@ import hash from "@src/data/hash"; export let metadata: IMetadata = null; export function start(): void { - metadata = { - sequence: 0, - version, - pageId: config.pageId || guid(), - userId: config.userId || guid(), - projectId: config.projectId || hash(location.host), - url: location.href, - title: document.title, - referrer: document.referrer - }; - + let pageId = config.pageId || guid(); + let userId = config.userId || guid(); + let projectId = config.projectId || hash(location.host); + let e: IEnvelope = { sequence: 0, version, pageId, userId, projectId, upload: Upload.Async }; + let p: IPage = { url: location.href, title: document.title, referrer: document.referrer }; + metadata = { page: p, envelope: e }; encode(); } @@ -25,9 +20,10 @@ export function end(): void { metadata = null; } -export function envelope(): Token[] { - metadata.sequence++; - return [metadata.sequence, metadata.version, metadata.pageId, metadata.userId, metadata.projectId]; +export function envelope(upload: Upload): Token[] { + let e = metadata.envelope; + if (upload !== Upload.Backup) { e.sequence++; } + return [e.sequence, e.version, e.pageId, e.userId, e.projectId, upload]; } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript diff --git a/src/data/queue.ts b/src/data/queue.ts deleted file mode 100644 index e2105516..00000000 --- a/src/data/queue.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Flush, Token } from "@clarity-types/data"; -import config from "@src/core/config"; -import upload from "@src/data/upload"; - -let events: Token[][] = []; -let timeout: number = null; - -window["PAYLOAD"] = []; - -export default function(data: Token[], flush: Flush = Flush.Schedule): void { - events.push(data); - - switch (flush) { - case Flush.Schedule: - clearTimeout(timeout); - timeout = window.setTimeout(dequeue, config.delay); - break; - case Flush.Force: - clearTimeout(timeout); - dequeue(); - break; - } -} - -function dequeue(): void { - upload(events); - reset(); -} - -function reset(): void { - events = []; -} diff --git a/src/data/upload.ts b/src/data/upload.ts index f94fd143..19c3981c 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,26 +1,91 @@ -import { IPayload, Token } from "@clarity-types/data"; +import { Event, IPayload, Token, Upload } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; import { counter } from "@src/metric"; import metrics from "@src/metric/encode"; -export default function(events: Token[][]): void { - let upload = config.upload ? config.upload : send; - let payload: IPayload = { e: envelope(), m: metrics(), d: events }; - let e = JSON.stringify(payload.e); - let m = JSON.stringify(payload.m); - let d = JSON.stringify(payload.d); - let data = `{"e":${e},"m":${m},"d":${d}}`; - counter(Metric.Bytes, data.length); - upload(data); +let events: string[]; +let timeout: number = null; + +export function start(): void { + events = []; + recover(); +} + +export function queue(data: Token[]): void { + let type = data.length > 1 ? data[1] : null; + let event = JSON.stringify(data); + events.push(event); + + switch (type) { + case Event.Discover: + case Event.Mutation: + case Event.BoxModel: + case Event.Checksum: + case Event.Document: + counter(Metric.LayoutBytes, event.length); + break; + case Event.Network: + case Event.Performance: + counter(Metric.NetworkBytes, event.length); + break; + case Event.ScriptError: + case Event.ImageError: + counter(Metric.DiagnosticBytes, event.length); + break; + default: + counter(Metric.InteractionBytes, event.length); + break; + } + + clearTimeout(timeout); + timeout = window.setTimeout(upload, config.delay); +} + +export function end(): void { + upload(true); + events = []; +} + +function upload(last: boolean = false): void { + let u = last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async; + let handler = config.upload ? config.upload : send; + let payload: IPayload = {e: JSON.stringify(envelope(u)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; + handler(stringify(payload), last); + backup(payload); + events = []; +} + +function stringify(payload: IPayload): string { + return `{"e":${payload.e},"m":${payload.m},"d":${payload.d}}`; } -function send(data: string): void { +function send(data: string, last: boolean = false): void { if (config.url.length > 0) { - let xhr = new XMLHttpRequest(); - xhr.open("POST", config.url); - xhr.setRequestHeader("Content-Type", "application/json"); - xhr.send(data); + if (last && "sendBeacon" in navigator) { + navigator.sendBeacon(config.url, data); + } else { + let xhr = new XMLHttpRequest(); + xhr.open("POST", config.url); + xhr.setRequestHeader("Content-Type", "application/json"); + xhr.send(data); + } + } +} + +function recover(): void { + if ("localStorage" in window) { + let data = localStorage.getItem("clarity-backup"); + if (data && data.length > 0) { + send(data); + } + } +} + +function backup(payload: IPayload): void { + if ("localStorage" in window) { + payload.e = JSON.stringify(envelope(Upload.Backup)); + localStorage.setItem("clarity-backup", stringify(payload)); } } diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts index 99098a7b..8dabd11e 100644 --- a/src/diagnostic/encode.ts +++ b/src/diagnostic/encode.ts @@ -1,7 +1,7 @@ import {Event, Token} from "@clarity-types/data"; import {Metric} from "@clarity-types/metric"; import time from "@src/core/time"; -import queue from "@src/data/queue"; +import { queue } from "@src/data/upload"; import * as image from "@src/diagnostic/image"; import * as script from "@src/diagnostic/script"; import * as metric from "@src/metric"; diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 1faf174e..b25dbfe6 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -1,17 +1,19 @@ import {Event, Token} from "@clarity-types/data"; import {Metric} from "@clarity-types/metric"; import time from "@src/core/time"; -import queue from "@src/data/queue"; +import { queue } from "@src/data/upload"; import * as metric from "@src/metric"; import * as change from "./change"; import * as mouse from "./mouse"; import * as resize from "./resize"; import * as scroll from "./scroll"; import * as selection from "./selection"; +import * as unload from "./unload"; import * as visibility from "./visibility"; export default function(type: Event): void { - let tokens: Token[] = [time(), type]; + let t = time(); + let tokens: Token[] = [t, type]; switch (type) { case Event.MouseDown: case Event.MouseUp: @@ -40,6 +42,13 @@ export default function(type: Event): void { metric.measure(Metric.ViewportHeight, r.height); resize.reset(); break; + case Event.Unload: + let u = unload.data; + tokens.push(u.name); + queue(tokens); + metric.counter(Metric.UnloadTime, t); + unload.reset(); + break; case Event.Change: let ch = change.data; tokens.push(ch.target); diff --git a/src/interaction/index.ts b/src/interaction/index.ts index 9cd48eaa..14327551 100644 --- a/src/interaction/index.ts +++ b/src/interaction/index.ts @@ -2,6 +2,7 @@ import * as mouse from "@src/interaction/mouse"; import * as resize from "@src/interaction/resize"; import * as scroll from "@src/interaction/scroll"; import * as selection from "@src/interaction/selection"; +import * as unload from "@src/interaction/unload"; import * as visibility from "@src/interaction/visibility"; export function start(): void { @@ -10,6 +11,7 @@ export function start(): void { visibility.start(); scroll.start(); selection.start(); + unload.start(); } export function end(): void { diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index 6af72d62..9c1e322b 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -6,7 +6,7 @@ import time from "@src/core/time"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -export let data: { [key: number]: IMouse[] } = []; +export let data: { [key: number]: IMouse[] } = {}; let timeout: number = null; export function start(): void { diff --git a/src/interaction/unload.ts b/src/interaction/unload.ts new file mode 100644 index 00000000..e6b8688d --- /dev/null +++ b/src/interaction/unload.ts @@ -0,0 +1,22 @@ +import { Event } from "@clarity-types/data"; +import { IUnload } from "@clarity-types/interaction"; +import { end } from "@src/clarity"; +import { bind } from "@src/core/event"; +import encode from "./encode"; + +export let data: IUnload; + +export function start(): void { + bind(window, "beforeunload", recompute); + bind(window, "unload", recompute); +} + +function recompute(evt: UIEvent): void { + data = { name: evt.type }; + encode(Event.Unload); + end(); +} + +export function reset(): void { + data = null; +} diff --git a/src/interaction/visibility.ts b/src/interaction/visibility.ts index aa92f5b1..2b96edda 100644 --- a/src/interaction/visibility.ts +++ b/src/interaction/visibility.ts @@ -1,9 +1,9 @@ import { Event } from "@clarity-types/data"; -import { IPageVisibility } from "@clarity-types/interaction"; +import { IVisibility } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; import encode from "./encode"; -export let data: IPageVisibility; +export let data: IVisibility; export function start(): void { bind(window, "pagehide", recompute); diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 7f67cf4a..4212401f 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -5,8 +5,8 @@ import mask from "@src/core/mask"; import * as task from "@src/core/task"; import time from "@src/core/time"; import hash from "@src/data/hash"; -import queue from "@src/data/queue"; import {check} from "@src/data/token"; +import { queue } from "@src/data/upload"; import * as metric from "@src/metric"; import * as boxmodel from "./boxmodel"; import {doc} from "./document"; diff --git a/src/metric/encode.ts b/src/metric/encode.ts index 90a5ea57..bddec750 100644 --- a/src/metric/encode.ts +++ b/src/metric/encode.ts @@ -2,7 +2,7 @@ import {Token} from "@clarity-types/data"; import { MetricType } from "@clarity-types/metric"; import { metrics, reset, updates } from "@src/metric"; -export default function(): Token[] { +export default function(last: boolean = false): Token[] { let output = []; // Encode counter metrics @@ -11,7 +11,7 @@ export default function(): Token[] { for (let metric in counters) { if (counters[metric]) { let m = num(metric); - if (updates.indexOf(m) >= 0) { + if (updates.indexOf(m) >= 0 || last) { output.push(m); output.push(counters[metric]); } @@ -24,7 +24,7 @@ export default function(): Token[] { for (let metric in measures) { if (measures[metric]) { let m = num(metric); - if (updates.indexOf(m) >= 0) { + if (updates.indexOf(m) >= 0 || last) { output.push(m); output.push(measures[metric]); } diff --git a/types/core.d.ts b/types/core.d.ts index ce18e010..0edc8450 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -25,7 +25,7 @@ export interface IConfig { lean?: boolean; tokens?: string[]; url?: string; - upload?: (data: string) => void; + upload?: (data: string, last: boolean) => void; } export interface ITaskTracker { diff --git a/types/data.d.ts b/types/data.d.ts index c1289b4f..10642933 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -4,7 +4,8 @@ export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); export const enum Event { - Metadata, + Page, + Unload, Discover, Mutation, BoxModel, @@ -30,16 +31,16 @@ export const enum Event { ImageError } -export const enum Flush { - Schedule, - Force, - None +export const enum Upload { + Async, + Beacon, + Backup } export interface IPayload { - e: Token[]; - m: Token[]; - d: Token[][]; + e: string; + m: string; + d: string; } export interface IDecodedPayload { @@ -54,16 +55,22 @@ export interface IDecodedEvent { data: any; } +export interface IMetadata { + page: IPage; + envelope: IEnvelope; +} + +export interface IPage { + url: string; + title: string; + referrer: string; +} + export interface IEnvelope { sequence: number; version: string; pageId: string; userId: string; projectId: string; -} - -export interface IMetadata extends IEnvelope { - url: string; - title: string; - referrer: string; + upload: Upload; } diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 5cb5daa4..1d141410 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -17,7 +17,7 @@ export interface IScroll { time?: number; } -export interface IPageVisibility { +export interface IVisibility { visible: string; } @@ -32,3 +32,7 @@ export interface IChange { target: number; value: string; } + +export interface IUnload { + name: string; +} diff --git a/types/metric.d.ts b/types/metric.d.ts index 85cad916..92681036 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -10,7 +10,10 @@ export const enum MetricType { export const enum Metric { /* Counter */ Nodes, - Bytes, + LayoutBytes, + InteractionBytes, + NetworkBytes, + DiagnosticBytes, Mutations, Interactions, Clicks, @@ -21,8 +24,9 @@ export const enum Metric { DiscoverTime, MutationTime, BoxModelTime, - WireupTime, + LoadTime, ActiveTime, + UnloadTime, /* Measures */ ViewportWidth, ViewportHeight, From 0668bddc356ec73b6d24079310ce6aa04135dd34 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 8 Sep 2019 15:05:57 -0700 Subject: [PATCH 084/105] Some rendering improvements --- decode/render.ts | 54 ++++++++++++++++++++-------------------- src/core/mask.ts | 3 ++- src/interaction/mouse.ts | 2 +- src/layout/dom.ts | 1 + src/layout/encode.ts | 1 + 5 files changed, 32 insertions(+), 29 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index be1f6a45..346882af 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -48,37 +48,36 @@ export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; for (let bm of data) { let el = element(bm.id) as HTMLElement; - switch (lean) { - case true: - let box = el ? el : doc.createElement("DIV"); - box.style.left = bm.box[0] + "px"; - box.style.top = bm.box[1] + "px"; - box.style.width = bm.box[2] + "px"; - box.style.height = bm.box[3] + "px"; - box.style.position = "absolute"; - box.style.border = "1px solid red"; - doc.body.appendChild(box); - nodes[bm.id] = box; - break; - case false: - if (el) { - let style = getComputedStyle(el, null); - let xPadding = parseInt(style["padding-left"], 10) + parseInt(style["padding-right"], 10); - let xBorder = parseInt(style["border-left"], 10) + parseInt(style["border-right"], 10); - let yPadding = parseInt(style["padding-top"], 10) + parseInt(style["padding-bottom"], 10); - let yBorder = parseInt(style["border-top"], 10) + parseInt(style["border-bottom"], 10); - let width = bm.box[2] - xPadding - xBorder; - el.style.maxWidth = width + "px"; - el.style.minWidth = Math.max(width - 5, 0) + "px"; - el.style.height = (bm.box[3] - yPadding - yBorder) + "px"; - el.style.overflow = "hidden"; - boxmodels[bm.id] = bm; - } - break; + if (lean) { + let box = el ? el : doc.createElement("DIV"); + box.style.left = bm.box[0] + "px"; + box.style.top = bm.box[1] + "px"; + box.style.width = bm.box[2] + "px"; + box.style.height = bm.box[3] + "px"; + box.style.position = "absolute"; + box.style.border = "1px solid red"; + doc.body.appendChild(box); + nodes[bm.id] = box; + } else { + let s = getComputedStyle(el, null); + let width = bm.box[2]; + let height = bm.box[3]; + if (s["boxSizing"] !== "border-box") { + width -= (css(s, "paddingLeft") + css(s, "paddingRight") + css(s, "borderLeftWidth") + css(s, "borderRightWidth")); + height -= (css(s, "paddingTop") + css(s, "paddingBottom") + css(s, "borderTopWidth") + css(s, "borderBottomWidth")); + } + el.style.width = width + "px"; + el.style.height = height + "px"; + if (el.tagName === "IFRAME") { el.style.backgroundColor = "maroon"; } + boxmodels[bm.id] = bm; } } } +function css(style: CSSStyleDeclaration, field: string): number { + return parseInt(style[field], 10); +} + export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; for (let node of data) { @@ -135,6 +134,7 @@ export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { setAttributes(styleElement as HTMLElement, node.attributes); styleElement.textContent = node.value; insert(node, parent, styleElement, next); + break; default: let domElement = element(node.id); domElement = domElement ? domElement : createElement(doc, node.tag, parent as HTMLElement); diff --git a/src/core/mask.ts b/src/core/mask.ts index a6b46dcb..ed0a1e2b 100644 --- a/src/core/mask.ts +++ b/src/core/mask.ts @@ -5,7 +5,8 @@ export default function(value: string): string { for (let i = 0; i < value.length; i++) { let code = value.charCodeAt(i); let isWhiteSpace = (code === 32 || code === 10 || code === 9 || code === 13); - textCount += isWhiteSpace ? 0 : 1; + let isNotCharacter = ((code >= 33 && code <= 47) || (code >= 91 && code <= 96) || (code >= 123 && code <= 126)); + textCount += isWhiteSpace || isNotCharacter ? 0 : 1; wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; wasWhiteSpace = isWhiteSpace; } diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts index 9c1e322b..68b45d55 100644 --- a/src/interaction/mouse.ts +++ b/src/interaction/mouse.ts @@ -46,7 +46,7 @@ function handler(event: Event, evt: MouseEvent): void { export function reset(): void { data = {}; - for (let event of [Event.MouseDown, Event.MouseUp, Event.MouseMove, Event.DoubleClick, Event.Click]) { + for (let event of [Event.MouseDown, Event.MouseUp, Event.MouseWheel, Event.MouseMove, Event.DoubleClick, Event.Click]) { data[event] = []; } } diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 5b9105d0..db9b1660 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -204,6 +204,7 @@ function leaf(tag: string, id: number, parentId: number): void { } break; case "IMG": + case "IFRAME": case "svg:svg": values[id].metadata.leaf = true; break; diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 4212401f..78905fb2 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -104,6 +104,7 @@ function meta(metadata: string[]): string[] | string[][] { function attribute(masked: boolean, key: string, value: string): string { switch (key) { case "src": + case "srcset": case "title": case "alt": return `${key}=${masked ? "" : value}`; From 7d28b58985a55d750af2faa17d2b817618b7536c Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 8 Sep 2019 21:26:44 -0700 Subject: [PATCH 085/105] Unmasking bugfix --- decode/layout.ts | 2 +- decode/render.ts | 2 +- src/layout/dom.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/decode/layout.ts b/decode/layout.ts index 32badf6a..3af78fad 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -104,7 +104,7 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let v = unmask(token.substr(keyIndex + 1)); switch (k) { case "src": - v = placeholderImage; + v = v.length === 0 ? placeholderImage : v; break; default: break; diff --git a/decode/render.ts b/decode/render.ts index 346882af..2110c3e0 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -58,7 +58,7 @@ export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { box.style.border = "1px solid red"; doc.body.appendChild(box); nodes[bm.id] = box; - } else { + } else if (el && el.tagName === "IFRAME") { let s = getComputedStyle(el, null); let width = bm.box[2]; let height = bm.box[3]; diff --git a/src/layout/dom.ts b/src/layout/dom.ts index db9b1660..4d9efd40 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -4,7 +4,7 @@ import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; const DEVTOOLS_HOOK: string = "__CLARITY_DEVTOOLS_HOOK__"; const MASK_ATTRIBUTE = "data-clarity-mask"; -const UNMASK_ATTRIBUTE = "data-clarity-umask"; +const UNMASK_ATTRIBUTE = "data-clarity-unmask"; let index: number = 1; let nodes: Node[] = []; From 68905d5d5397ca69828545be1a397754ae7dbc27 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 9 Sep 2019 08:23:48 -0700 Subject: [PATCH 086/105] Metric rendering bug fix and sending end signal --- decode/clarity.ts | 5 +++-- decode/envelope.ts | 3 ++- decode/metric.ts | 10 +++++++--- src/data/encode.ts | 2 +- src/data/metadata.ts | 7 ++++--- src/data/upload.ts | 7 +++---- src/interaction/encode.ts | 2 +- types/data.d.ts | 1 + types/metric.d.ts | 4 ++-- 9 files changed, 24 insertions(+), 17 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index c8883bff..c66c88e2 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -2,7 +2,7 @@ import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types import envelope from "./envelope"; import interaction from "./interaction"; import layout from "./layout"; -import metric from "./metric"; +import * as m from "./metric"; import page from "./page"; import * as r from "./render"; @@ -13,7 +13,7 @@ export function json(data: string): IDecodedPayload { let payload = JSON.parse(data); let decoded: IDecodedPayload = { envelope: envelope(payload.e), - metrics: metric(payload.m), + metrics: m.metric(payload.m), events: [] }; @@ -21,6 +21,7 @@ export function json(data: string): IDecodedPayload { payloads = []; pageId = decoded.envelope.pageId; r.reset(); + m.reset(); } let encoded: Token[][] = payload.d; diff --git a/decode/envelope.ts b/decode/envelope.ts index 22a8cd90..ca252937 100644 --- a/decode/envelope.ts +++ b/decode/envelope.ts @@ -7,6 +7,7 @@ export default function(tokens: Token[]): IEnvelope { pageId: tokens[2] as string, userId: tokens[3] as string, projectId: tokens[4] as string, - upload: tokens[5] as Upload + upload: tokens[5] as Upload, + end: tokens[6] as number }; } diff --git a/decode/metric.ts b/decode/metric.ts index bcb74110..e4655982 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -17,9 +17,9 @@ metricMap[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; metricMap[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; metricMap[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; metricMap[Metric.BoxModelTime] = { name: "Box Model Time", unit: "ms"}; -metricMap[Metric.LoadTime] = { name: "Load Time", unit: "s"}; +metricMap[Metric.StartTime] = { name: "Start Time", unit: "s"}; metricMap[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; -metricMap[Metric.UnloadTime] = { name: "Unload Time", unit: "s"}; +metricMap[Metric.EndTime] = { name: "End Time", unit: "s"}; metricMap[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px"}; metricMap[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; metricMap[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; @@ -27,7 +27,7 @@ metricMap[Metric.DocumentHeight] = { name: "Document Height", unit: "px"}; let metrics: IDecodedMetric = { counters: {}, measures: {}, events: [], marks: [] }; -export default function(tokens: Token[]): IDecodedMetric { +export function metric(tokens: Token[]): IDecodedMetric { let i = 0; let metricType = null; while (i < tokens.length) { @@ -58,3 +58,7 @@ export default function(tokens: Token[]): IDecodedMetric { return metrics; } + +export function reset(): void { + metrics = { counters: {}, measures: {}, events: [], marks: [] }; +} diff --git a/src/data/encode.ts b/src/data/encode.ts index 8f4ed148..b7951ddd 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -8,7 +8,7 @@ import { queue } from "./upload"; export default function(): void { let t = time(); let tokens: Token[] = [t, Event.Page]; - metric.counter(Metric.LoadTime, t); + metric.counter(Metric.StartTime, t); tokens.push(metadata.page.url); tokens.push(metadata.page.title); tokens.push(metadata.page.referrer); diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 2469fc63..c403777a 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -10,7 +10,7 @@ export function start(): void { let pageId = config.pageId || guid(); let userId = config.userId || guid(); let projectId = config.projectId || hash(location.host); - let e: IEnvelope = { sequence: 0, version, pageId, userId, projectId, upload: Upload.Async }; + let e: IEnvelope = { sequence: 0, version, pageId, userId, projectId, upload: Upload.Async, end: 0 }; let p: IPage = { url: location.href, title: document.title, referrer: document.referrer }; metadata = { page: p, envelope: e }; encode(); @@ -20,10 +20,11 @@ export function end(): void { metadata = null; } -export function envelope(upload: Upload): Token[] { +export function envelope(last: boolean, backup: boolean = false): Token[] { + let upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); let e = metadata.envelope; if (upload !== Upload.Backup) { e.sequence++; } - return [e.sequence, e.version, e.pageId, e.userId, e.projectId, upload]; + return [e.sequence, e.version, e.pageId, e.userId, e.projectId, upload, last ? 1 : 0]; } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript diff --git a/src/data/upload.ts b/src/data/upload.ts index 19c3981c..d24ebfe1 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,4 +1,4 @@ -import { Event, IPayload, Token, Upload } from "@clarity-types/data"; +import { Event, IPayload, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; @@ -49,9 +49,8 @@ export function end(): void { } function upload(last: boolean = false): void { - let u = last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async; let handler = config.upload ? config.upload : send; - let payload: IPayload = {e: JSON.stringify(envelope(u)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; + let payload: IPayload = {e: JSON.stringify(envelope(last)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; handler(stringify(payload), last); backup(payload); events = []; @@ -85,7 +84,7 @@ function recover(): void { function backup(payload: IPayload): void { if ("localStorage" in window) { - payload.e = JSON.stringify(envelope(Upload.Backup)); + payload.e = JSON.stringify(envelope(true, true)); localStorage.setItem("clarity-backup", stringify(payload)); } } diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index b25dbfe6..9c46e19a 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -46,7 +46,7 @@ export default function(type: Event): void { let u = unload.data; tokens.push(u.name); queue(tokens); - metric.counter(Metric.UnloadTime, t); + metric.counter(Metric.EndTime, t); unload.reset(); break; case Event.Change: diff --git a/types/data.d.ts b/types/data.d.ts index 10642933..4e1424f4 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -73,4 +73,5 @@ export interface IEnvelope { userId: string; projectId: string; upload: Upload; + end: number; } diff --git a/types/metric.d.ts b/types/metric.d.ts index 92681036..771a4bea 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -24,9 +24,9 @@ export const enum Metric { DiscoverTime, MutationTime, BoxModelTime, - LoadTime, + StartTime, ActiveTime, - UnloadTime, + EndTime, /* Measures */ ViewportWidth, ViewportHeight, From 546a373d1688ead4eb8e4eaf9f099d26555c7928 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 9 Sep 2019 19:54:02 -0700 Subject: [PATCH 087/105] Handling dom removals --- decode/render.ts | 1 + src/layout/dom.ts | 12 +++++++++++- src/layout/encode.ts | 7 ++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/decode/render.ts b/decode/render.ts index 2110c3e0..2f4fa422 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -170,6 +170,7 @@ function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void } } else if (parent === null && node.parentElement !== null) { node.parentElement.removeChild(node); + node = null; } nodes[data.id] = node; } diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 4d9efd40..65fa8311 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -68,6 +68,7 @@ export function update(node: Node, data: INodeData, source: Source): void { if (id in values) { let value = values[id]; + value.metadata.active = true; // Handle case where internal ordering may have changed if (value["next"] !== nextId) { @@ -87,7 +88,7 @@ export function update(node: Node, data: INodeData, source: Source): void { } } else { // Mark this element as deleted if the parent has been updated to null - value.metadata.active = false; + remove(id, source); } // Remove reference to this node from the old parent @@ -169,6 +170,15 @@ export function selectors(): INodeValue[] { return v; } +function remove(id: number, source: Source): void { + let value = values[id]; + value.metadata.active = false; + value.parent = null; + track(id, source); + for (let child of value.children) { remove(child, source); } + value.children = []; +} + function selector(id: number, data: INodeData, parent: string): string { switch (data.tag) { case "STYLE": diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 78905fb2..3fb18081 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -52,15 +52,16 @@ export default async function(type: Event): Promise { if (task.longtask(timer)) { await task.idle(timer); } let metadata = []; let data: INodeData = value.data; - let keys = ["tag", "path", "attributes", "value"]; + let active = value.metadata.active; + let keys = active ? ["tag", "path", "attributes", "value"] : ["tag"]; for (let key of keys) { if (data[key]) { switch (key) { case "tag": metric.counter(Metric.Nodes); tokens.push(value.id); - if (value.parent) { tokens.push(value.parent); } - if (value.next) { tokens.push(value.next); } + if (value.parent && active) { tokens.push(value.parent); } + if (value.next && active) { tokens.push(value.next); } metadata.push(data[key]); break; case "path": From 4261f79d834e63056edf347d56a081563a0b0994 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 10 Sep 2019 07:19:56 -0700 Subject: [PATCH 088/105] Stateless decoding --- decode/clarity.ts | 28 ++++++++++------------------ decode/metric.ts | 9 +++------ decode/render.ts | 24 ++++++++++++++++++------ 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index c66c88e2..50690429 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,31 +1,17 @@ -import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; +import { Event, IDecodedEvent, IDecodedPayload, Token } from "../types/data"; import envelope from "./envelope"; import interaction from "./interaction"; import layout from "./layout"; -import * as m from "./metric"; +import metric from "./metric"; import page from "./page"; import * as r from "./render"; let pageId: string = null; -let payloads: IPayload[] = []; export function json(data: string): IDecodedPayload { let payload = JSON.parse(data); - let decoded: IDecodedPayload = { - envelope: envelope(payload.e), - metrics: m.metric(payload.m), - events: [] - }; - - if (pageId !== decoded.envelope.pageId) { - payloads = []; - pageId = decoded.envelope.pageId; - r.reset(); - m.reset(); - } - + let decoded: IDecodedPayload = { envelope: envelope(payload.e), metrics: metric(payload.m), events: [] }; let encoded: Token[][] = payload.d; - payloads.push(payload); for (let entry of encoded) { let event: IDecodedEvent; @@ -70,8 +56,14 @@ export function html(decoded: IDecodedPayload): string { } export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, header?: HTMLElement): void { + // Reset rendering if we receive a new pageId + if (pageId !== decoded.envelope.pageId) { + pageId = decoded.envelope.pageId; + r.reset(); + } + // Render metrics - r.metrics(decoded.metrics, header); + r.metric(decoded.metrics, header); // Replay events replay(decoded.events, iframe); diff --git a/decode/metric.ts b/decode/metric.ts index e4655982..351621b8 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -25,11 +25,12 @@ metricMap[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; metricMap[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; metricMap[Metric.DocumentHeight] = { name: "Document Height", unit: "px"}; -let metrics: IDecodedMetric = { counters: {}, measures: {}, events: [], marks: [] }; +let metrics: IDecodedMetric = null; -export function metric(tokens: Token[]): IDecodedMetric { +export default function(tokens: Token[]): IDecodedMetric { let i = 0; let metricType = null; + metrics = { counters: {}, measures: {}, events: [], marks: [] }; while (i < tokens.length) { // Determine metric time for subsequent processing if (typeof(tokens[i]) === "string") { @@ -58,7 +59,3 @@ export function metric(tokens: Token[]): IDecodedMetric { return metrics; } - -export function reset(): void { - metrics = { counters: {}, measures: {}, events: [], marks: [] }; -} diff --git a/decode/render.ts b/decode/render.ts index 2f4fa422..5404a6ef 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -5,6 +5,7 @@ import { IDecodedMetric } from "../types/metric"; let nodes = {}; let boxmodels = {}; +let metrics: IDecodedMetric = null; let svgns: string = "http://www.w3.org/2000/svg"; let lean = false; @@ -16,16 +17,27 @@ let clickIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S export function reset(): void { nodes = {}; boxmodels = {}; + metrics = { counters: {}, measures: {}, events: [], marks: [] }; } -export function metrics(data: IDecodedMetric, header: HTMLElement): void { +export function metric(data: IDecodedMetric, header: HTMLElement): void { let html = []; - let entries = {...data.counters, ...data.measures}; - for (let metric in entries) { - if (entries[metric]) { - let m = entries[metric]; - html.push(`

  • ${value(m.value, m.unit)}${m.unit}

    ${metric}
    • `); + // Copy over counters + for (let counter in data.counters) { + if (data.counters[counter]) { metrics.counters[counter] = data.counters[counter]; } + } + + // Copy over measures + for (let measure in data.measures) { + if (data.measures[measure]) { metrics.measures[measure] = data.measures[measure]; } + } + + let entries = {...metrics.counters, ...metrics.measures}; + for (let entry in entries) { + if (entries[entry]) { + let m = entries[entry]; + html.push(`

  • ${value(m.value, m.unit)}${m.unit}

    ${entry}
    • `); } } From edbd844a2e0591b085ed8a2a29968cf9ccb4fcde Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 10 Sep 2019 08:28:28 -0700 Subject: [PATCH 089/105] Splitting events into 3 streams at decoding time --- decode/clarity.ts | 31 ++++++++++++++++++++++--------- decode/summary.ts | 21 +++++++++++++++++++++ types/data.d.ts | 7 +++++-- types/layout.d.ts | 4 ++++ 4 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 decode/summary.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 50690429..23dca1c8 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -5,16 +5,18 @@ import layout from "./layout"; import metric from "./metric"; import page from "./page"; import * as r from "./render"; +import summarize from "./summary"; let pageId: string = null; -export function json(data: string): IDecodedPayload { - let payload = JSON.parse(data); - let decoded: IDecodedPayload = { envelope: envelope(payload.e), metrics: metric(payload.m), events: [] }; - let encoded: Token[][] = payload.d; +export function decode(data: string): IDecodedPayload { + let json = JSON.parse(data); + let payload: IDecodedPayload = { envelope: envelope(json.e), metrics: metric(json.m), analytics: [], playback: [], summary: [] }; + let encoded: Token[][] = json.d; for (let entry of encoded) { let event: IDecodedEvent; + let summary: IDecodedEvent; switch (entry[1]) { case Event.Scroll: case Event.Document: @@ -29,24 +31,34 @@ export function json(data: string): IDecodedPayload { case Event.DoubleClick: case Event.RightClick: event = interaction(entry); + payload.analytics.push(event); + break; + case Event.BoxModel: + event = layout(entry); + payload.playback.push(event); break; case Event.Discover: case Event.Mutation: - case Event.BoxModel: + event = layout(entry); + summary = summarize(event); + payload.playback.push(event); + payload.summary.push(summary); + break; case Event.Checksum: event = layout(entry); + payload.summary.push(event); break; case Event.Page: event = page(entry); + payload.analytics.push(event); break; default: event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; + payload.playback.push(event); break; } - decoded.events.push(event); } - decoded.events.sort(sort); - return decoded; + return payload; } export function html(decoded: IDecodedPayload): string { @@ -66,7 +78,8 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head r.metric(decoded.metrics, header); // Replay events - replay(decoded.events, iframe); + let events = [...decoded.analytics, ...decoded.playback, ...decoded.summary].sort(sort); + replay(events, iframe); } export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement): Promise { diff --git a/decode/summary.ts b/decode/summary.ts new file mode 100644 index 00000000..66b1f860 --- /dev/null +++ b/decode/summary.ts @@ -0,0 +1,21 @@ +import hash from "../src/data/hash"; +import { Event, IDecodedEvent } from "../types/data"; +import { IChecksumMap, IDecodedNode } from "../types/layout"; + +export default function(event: IDecodedEvent): IDecodedEvent { + switch (event.event) { + case Event.Discover: + case Event.Mutation: + let checksumMap: IDecodedEvent = {time: event.time, event: Event.ChecksumMap, data: []}; + let nodes: IDecodedNode[] = event.data; + for (let node of nodes) { + // Do not track nodes where we don't have a valid selector - e.g. text nodes + if (node.selector && node.selector.length > 0) { + let checksum = hash(node.selector); + let data: IChecksumMap = { id: node.id, checksum, selector: node.selector }; + checksumMap.data.push(data); + } + } + return checksumMap; + } +} diff --git a/types/data.d.ts b/types/data.d.ts index 4e1424f4..54c78439 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -28,7 +28,8 @@ export const enum Event { Network, Performance, ScriptError, - ImageError + ImageError, + ChecksumMap } export const enum Upload { @@ -46,7 +47,9 @@ export interface IPayload { export interface IDecodedPayload { envelope: IEnvelope; metrics: IDecodedMetric; - events: IDecodedEvent[]; + analytics: IDecodedEvent[]; + playback: IDecodedEvent[]; + summary: IDecodedEvent[]; } export interface IDecodedEvent { diff --git a/types/layout.d.ts b/types/layout.d.ts index 13ff464c..9d8093ca 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -63,3 +63,7 @@ export interface IChecksum { id: number; checksum: string; } + +export interface IChecksumMap extends IChecksum { + selector: string; +} From 39eb5c0fa065a97f0a7c28032a0b4fccea7a07a6 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Tue, 10 Sep 2019 09:18:55 -0700 Subject: [PATCH 090/105] Renaming ChecksumMap to LayoutSummary --- decode/summary.ts | 6 +++--- types/data.d.ts | 2 +- types/layout.d.ts | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/decode/summary.ts b/decode/summary.ts index 66b1f860..50442510 100644 --- a/decode/summary.ts +++ b/decode/summary.ts @@ -1,18 +1,18 @@ import hash from "../src/data/hash"; import { Event, IDecodedEvent } from "../types/data"; -import { IChecksumMap, IDecodedNode } from "../types/layout"; +import { IDecodedNode, ILayoutSummary } from "../types/layout"; export default function(event: IDecodedEvent): IDecodedEvent { switch (event.event) { case Event.Discover: case Event.Mutation: - let checksumMap: IDecodedEvent = {time: event.time, event: Event.ChecksumMap, data: []}; + let checksumMap: IDecodedEvent = {time: event.time, event: Event.LayoutSummary, data: []}; let nodes: IDecodedNode[] = event.data; for (let node of nodes) { // Do not track nodes where we don't have a valid selector - e.g. text nodes if (node.selector && node.selector.length > 0) { let checksum = hash(node.selector); - let data: IChecksumMap = { id: node.id, checksum, selector: node.selector }; + let data: ILayoutSummary = { id: node.id, checksum, selector: node.selector }; checksumMap.data.push(data); } } diff --git a/types/data.d.ts b/types/data.d.ts index 54c78439..f3a351fd 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -29,7 +29,7 @@ export const enum Event { Performance, ScriptError, ImageError, - ChecksumMap + LayoutSummary } export const enum Upload { diff --git a/types/layout.d.ts b/types/layout.d.ts index 9d8093ca..2665745f 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -64,6 +64,8 @@ export interface IChecksum { checksum: string; } -export interface IChecksumMap extends IChecksum { +export interface ILayoutSummary { + id: number; + checksum: string; selector: string; } From 80b938001242789ae4109709bd3335a58debb0e4 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 11 Sep 2019 06:10:31 -0700 Subject: [PATCH 091/105] Simplifying unique ids and cookie tracking --- decode/clarity.ts | 3 ++- decode/envelope.ts | 7 +++--- src/clarity.ts | 34 ++++++++++++++------------ src/core/config.ts | 4 ++-- src/core/index.ts | 4 ++++ src/data/metadata.ts | 57 ++++++++++++++++++++++++++++---------------- src/layout/dom.ts | 18 +++++++------- types/core.d.ts | 4 ++-- types/data.d.ts | 8 +++++++ 9 files changed, 88 insertions(+), 51 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 23dca1c8..21be13a4 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -11,7 +11,8 @@ let pageId: string = null; export function decode(data: string): IDecodedPayload { let json = JSON.parse(data); - let payload: IDecodedPayload = { envelope: envelope(json.e), metrics: metric(json.m), analytics: [], playback: [], summary: [] }; + let time = Date.now(); + let payload: IDecodedPayload = { time, envelope: envelope(json.e), metrics: metric(json.m), analytics: [], playback: [], summary: [] }; let encoded: Token[][] = json.d; for (let entry of encoded) { diff --git a/decode/envelope.ts b/decode/envelope.ts index ca252937..53373f09 100644 --- a/decode/envelope.ts +++ b/decode/envelope.ts @@ -6,8 +6,9 @@ export default function(tokens: Token[]): IEnvelope { version: tokens[1] as string, pageId: tokens[2] as string, userId: tokens[3] as string, - projectId: tokens[4] as string, - upload: tokens[5] as Upload, - end: tokens[6] as number + sessionId: tokens[4] as string, + projectId: tokens[5] as string, + upload: tokens[6] as Upload, + end: tokens[7] as number }; } diff --git a/src/clarity.ts b/src/clarity.ts index 2da55edc..daa8bb8a 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -10,26 +10,30 @@ let status = false; /* Initial discovery of DOM */ export function start(configuration: IConfig = {}): void { - core.start(configuration); - metric.start(); - data.start(); - diagnostic.start(); - dom.start(); - interaction.start(); + if (core.check()) { + core.start(configuration); + metric.start(); + data.start(); + diagnostic.start(); + dom.start(); + interaction.start(); - // Mark Clarity session as active - status = true; + // Mark Clarity session as active + status = true; + } } export function end(): void { - interaction.end(); - dom.end(); - diagnostic.end(); - data.end(); - metric.end(); - core.end(); + if (status) { + interaction.end(); + dom.end(); + diagnostic.end(); + data.end(); + metric.end(); + core.end(); - status = false; + status = false; + } } export function active(): boolean { diff --git a/src/core/config.ts b/src/core/config.ts index e503f2c3..d6aa5e56 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,14 +1,14 @@ import { IConfig } from "@clarity-types/core"; let config: IConfig = { - pageId: null, - userId: null, + sessionId: null, projectId: null, longTask: 30, lookahead: 500, distance: 20, interval: 25, delay: 1000, + expire: 7, cssRules: false, lean: false, tokens: [], diff --git a/src/core/index.ts b/src/core/index.ts index a50bc9a8..d9eba4e7 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -14,3 +14,7 @@ export function start(configuration: IConfig): void { export function end(): void { event.reset(); } + +export function check(): boolean { + return window["MutationObserver"] && document["createTreeWalker"] && "now" in Date ? true : false; +} diff --git a/src/data/metadata.ts b/src/data/metadata.ts index c403777a..496d6a3c 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,18 +1,25 @@ -import { IEnvelope, IMetadata, IPage, Token, Upload } from "@clarity-types/data"; +import { ICookieData, IEnvelope, IMetadata, IPage, Token, Upload } from "@clarity-types/data"; import config from "@src/core/config"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; +const CLARITY_COOKIE_NAME: string = "_clarity"; +const CLARITY_COOKIE_SEPARATOR: string = "|"; +const CLARITY_SESSION_LENGTH = 30 * 60 * 1000; export let metadata: IMetadata = null; export function start(): void { - let pageId = config.pageId || guid(); - let userId = config.userId || guid(); + let cookie: ICookieData = read(); + let timestamp = Date.now(); + let pageId = id(); let projectId = config.projectId || hash(location.host); - let e: IEnvelope = { sequence: 0, version, pageId, userId, projectId, upload: Upload.Async, end: 0 }; + let userId = cookie && cookie.userId ? cookie.userId : id(); + let sessionId = cookie && cookie.sessionId && timestamp - cookie.timestamp < CLARITY_SESSION_LENGTH ? cookie.sessionId : id(); + let e: IEnvelope = { sequence: 0, version, pageId, userId, sessionId, projectId, upload: Upload.Async, end: 0 }; let p: IPage = { url: location.href, title: document.title, referrer: document.referrer }; metadata = { page: p, envelope: e }; + track({ userId, sessionId, timestamp }); encode(); } @@ -24,23 +31,33 @@ export function envelope(last: boolean, backup: boolean = false): Token[] { let upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); let e = metadata.envelope; if (upload !== Upload.Backup) { e.sequence++; } - return [e.sequence, e.version, e.pageId, e.userId, e.projectId, upload, last ? 1 : 0]; + return [e.sequence, e.version, e.pageId, e.userId, e.sessionId, e.projectId, upload, last ? 1 : 0]; } -// Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript -// Excluding 3rd party code from tslint -// tslint:disable -function guid() { - let d = new Date().getTime(); - if (window.performance && performance.now) { - // Use high-precision timer if available - d += performance.now(); +function id(): string { + return Math.random().toString(36).slice(-6) + Date.now().toString(36).slice(-4); +} + +function track(data: ICookieData): void { + let expiry = new Date(); + expiry.setDate(expiry.getDate() + config.expire); + let expires = expiry ? "expires=" + expiry.toUTCString() : ""; + let value = `${data.userId}|${data.sessionId}|${data.timestamp}` + ";" + expires + ";path=/"; + document.cookie = CLARITY_COOKIE_NAME + "=" + value; +} + +function read(): ICookieData { + let cookies: string[] = document.cookie.split(";"); + if (cookies) { + for (let i = 0; i < cookies.length; i++) { + let pair: string[] = cookies[i].split("="); + if (pair.length > 1 && pair[0].indexOf(CLARITY_COOKIE_NAME) >= 0 && pair[1].indexOf(CLARITY_COOKIE_SEPARATOR) > 0) { + let parts = pair[1].split(CLARITY_COOKIE_SEPARATOR); + if (parts.length === 3) { + return { userId: parts[0], sessionId: parts[1], timestamp: parseInt(parts[2], 10) }; + } + } } - let uuid = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, function(c) { - let r = (d + Math.random() * 16) % 16 | 0; - d = Math.floor(d / 16); - return (c == "x" ? r : (r & 0x3 | 0x8)).toString(16); - }); - return uuid; + } + return null; } -// tslint:enable diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 65fa8311..914c0ae0 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -202,14 +202,16 @@ function leaf(tag: string, id: number, parentId: number): void { if (id !== null && parentId !== null) { switch (tag) { case "*T": - // Mark parent as a leaf node only if the text node has valid text - // For nodes with whitespaces and not real text, skip them - let value = values[id].data.value; - for (let i = 0; i < value.length; i++) { - let code = value.charCodeAt(i); - if (!(code === 32 || code === 10 || code === 9 || code === 13)) { - values[parentId].metadata.leaf = true; - break; + // Mark parent as a leaf node only if the text node has valid text and parent is masked. + // For nodes with whitespaces and not real text, skip them. + if (values[parentId].metadata.masked) { + let value = values[id].data.value; + for (let i = 0; i < value.length; i++) { + let code = value.charCodeAt(i); + if (!(code === 32 || code === 10 || code === 9 || code === 13)) { + values[parentId].metadata.leaf = true; + break; + } } } break; diff --git a/types/core.d.ts b/types/core.d.ts index 0edc8450..ba69450c 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -13,14 +13,14 @@ export interface IBindingContainer { } export interface IConfig { - pageId?: string; - userId?: string; + sessionId?: string; projectId?: string; longTask?: number; lookahead?: number; distance?: number; interval?: number; delay?: number; + expire?: number; cssRules?: boolean; lean?: boolean; tokens?: string[]; diff --git a/types/data.d.ts b/types/data.d.ts index f3a351fd..0034cf54 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -45,6 +45,7 @@ export interface IPayload { } export interface IDecodedPayload { + time: number; envelope: IEnvelope; metrics: IDecodedMetric; analytics: IDecodedEvent[]; @@ -58,6 +59,12 @@ export interface IDecodedEvent { data: any; } +export interface ICookieData { + userId: string; + sessionId: string; + timestamp: number; +} + export interface IMetadata { page: IPage; envelope: IEnvelope; @@ -74,6 +81,7 @@ export interface IEnvelope { version: string; pageId: string; userId: string; + sessionId: string; projectId: string; upload: Upload; end: number; From df121c6aeca66fa65ea2304a3d6cc96247a8d81f Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 11 Sep 2019 08:05:26 -0700 Subject: [PATCH 092/105] Refactor pointer and support for custom selector --- decode/clarity.ts | 12 +++++- decode/interaction.ts | 10 +++-- decode/layout.ts | 3 ++ decode/render.ts | 44 ++++++++++++-------- src/interaction/encode.ts | 28 +++++++------ src/interaction/index.ts | 4 +- src/interaction/mouse.ts | 58 -------------------------- src/interaction/pointer.ts | 84 ++++++++++++++++++++++++++++++++++++++ src/interaction/touch.ts | 0 src/layout/boxmodel.ts | 19 +-------- src/layout/dom.ts | 28 +++++++++---- types/data.d.ts | 6 ++- types/interaction.d.ts | 2 +- types/layout.d.ts | 2 +- 14 files changed, 176 insertions(+), 124 deletions(-) delete mode 100644 src/interaction/mouse.ts create mode 100644 src/interaction/pointer.ts delete mode 100644 src/interaction/touch.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 21be13a4..7741e95c 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -31,6 +31,10 @@ export function decode(data: string): IDecodedPayload { case Event.Click: case Event.DoubleClick: case Event.RightClick: + case Event.TouchStart: + case Event.TouchCancel: + case Event.TouchEnd: + case Event.TouchMove: event = interaction(entry); payload.analytics.push(event); break; @@ -106,7 +110,13 @@ export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement) case Event.Click: case Event.DoubleClick: case Event.RightClick: - r.mouse(entry.event, entry.data, iframe); + r.pointer(entry.event, entry.data, iframe); + break; + case Event.TouchStart: + case Event.TouchCancel: + case Event.TouchEnd: + case Event.TouchMove: + r.pointer(entry.event, entry.data, iframe); break; case Event.Change: r.change(entry.data, iframe); diff --git a/decode/interaction.ts b/decode/interaction.ts index ac70e9fb..f162d2a3 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -1,5 +1,5 @@ import { Event, IDecodedEvent, Token } from "../types/data"; -import { IChange, IMouse, IResize, IScroll, ISelection } from "../types/interaction"; +import { IChange, IPointer, IResize, IScroll, ISelection } from "../types/interaction"; export default function(tokens: Token[]): IDecodedEvent { let time = tokens[0] as number; @@ -12,8 +12,12 @@ export default function(tokens: Token[]): IDecodedEvent { case Event.Click: case Event.DoubleClick: case Event.RightClick: - let mouseData: IMouse = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; - return { time, event, data: mouseData }; + case Event.TouchStart: + case Event.TouchCancel: + case Event.TouchEnd: + case Event.TouchMove: + let pointerData: IPointer = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; + return { time, event, data: pointerData }; case Event.Resize: let resizeData: IResize = { width: tokens[2] as number, height: tokens[3] as number }; return { time, event, data: resizeData }; diff --git a/decode/layout.ts b/decode/layout.ts index 3af78fad..255aba14 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -2,6 +2,7 @@ import { resolve } from "../src/data/token"; import { Event, IDecodedEvent, Token } from "../types/data"; import { IAttributes, IBoxModel, IChecksum, IDecodedNode, IDocumentSize, } from "../types/layout"; +const ID_ATTRIBUTE = "data-clarity"; let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; let selectorMap = {}; @@ -129,11 +130,13 @@ function selector(id: number, path: string, tag: string, attributes: IAttributes case "LINK": case "META": case "*T": + case "*D": return ""; default: let s = path && path.length > 0 ? path + tag : tag; if ("id" in attributes) { s = `${tag}#${attributes["id"]}`; } if ("class" in attributes) { s += `.${attributes["class"].trim().split(" ").join(".")}`; } + if (ID_ATTRIBUTE in attributes) { s = `*${attributes[ID_ATTRIBUTE]}`; } selectorMap[id] = s; return s; } diff --git a/decode/render.ts b/decode/render.ts index 5404a6ef..4d4e7cd4 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,5 +1,5 @@ import { Event } from "../types/data"; -import { IChange, IMouse, IResize, IScroll, ISelection } from "../types/interaction"; +import { IChange, IPointer, IResize, IScroll, ISelection } from "../types/interaction"; import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; import { IDecodedMetric } from "../types/metric"; @@ -13,6 +13,8 @@ let lean = false; let pointerIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S912AAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6JQAAgIMAAPn/AACA6AAAdTAAAOpgAAA6lwAAF2+XqZnUAAACaUlEQVR4nGL8f58BHYgAsT8Q2wGxBBAzQcX/AfFrID4CxOuA+BWKLoX/YAoggBjRDHQD4ngglgRiPgyrIOAzEL8E4lVQg1EMBAggFiSFYUAcA8RSOAyCAV4oTgViTiBeiiwJEEAw71gRaRgyEAXiKCB2RBYECCCQgcIMEG+SYhgMiANxEhDzwwQAAghkoAMQK5NhGAwoALE1jAMQQCADQU7mpMBAZqijwAAggEAGqgAxOwUGskHNAAOAAAIZyEtIh4INg3bfHHD6xAUEYAyAAAIZ+IuQgU9fMLCXdzDIzV3JIIhDyQ8YAyCAQAaCUv8/fAZysDP8+/OXgTG7jkFhwRoMQ0F6n8M4AAEEMvAKsg34wM9fDEwgQ1dtRSQTIPgNxFdhHIAAAhm4AYg/EmMgCHz7zsCUVMaguHob3FCQYzbD5AECCGTgJSDeCbWJKPD1GwNzSjmD4tZ9DFxgvQr/b8PkAAIIlvVWA/FuUgz99IWBOTyXQcE+nOEOsjhAACGXNnJAHAnE9kAshqyIV5vB4Ms3cALGBkAlj9////9PgTgAAcSEJPEIiDuBeBYQP2CAhOt3BsLJCpSfNzAyMpqDOAABhF4ewh3FAMmf2kAsyqnBUPDjJ8HcdBvoSjWAAGIEEgTUMTAAbf/AwICSVGCgD4hPgJQA8WegWdsBAogFiyJC4C0QgxI3KLj4gIasRpYECCAGkAsJYSAAuRDEAKUEQwZIzgDxvwCxCrJagAAi1kAQAYpFESh/BlQMhJuR1QIEELEGlgOxHBLflAGSh0Gc60DMBpMDCCCiDMRhyXoGSJUaDgpPmDhAgAEAN5Ugk0bMYNIAAAAASUVORK5CYII="; // tslint:disable-next-line: max-line-length let clickIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S912AAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6JQAAgIMAAPn/AACA6AAAdTAAAOpgAAA6lwAAF2+XqZnUAAADvklEQVR4nGL8//8/AzJgZGQQAVL+QGwHxBJAzASV+gfEr4H4CBCvA2p7xYAFAAQQI7KBQMPcgFQ8EEsCMR82DUDwGYhfAvEqoNZ16JIAAQQ3EGhYGJCKAWIpHAahA5BrlwC1L0UWBAggJqhhViQaBgKiQBwF1OuILAgQQExAAWEGiDeRDJsEFOM1AFplzMCgrcnAcIoTh6HiQJwENIMfJgAQQCAXOgCxMkLNaqBkvgIDg8Z3BoaC5wwMz9kYGKIVGRg+MjFgB0C1DNYwDkAAgRSBnIzkgnUCDAyswIBdf4+Bof8ZA0PHYwaGO0D5/Tw4DGSGOgoMAAIIZKAKELMj5D8AFXD+BUbyXwhf9Scw5QAt+MmIw0A2qBlgABBAIAN5cSgkBQjAGAABxALEv1BdiA4YobgFmDZPcjMwZLyB+JILmNAl/0AV/YCpBgggkIGg9MTNgMgRWAAL0MuvWRkYJgNzznRgzP4AqmUHGpj/AhgFnxgYlD4yMKiBVQIEEMiQK8g2YAJQ2JUAY/vFZQaGmfeBvgO6qhUYUU5AQ7qASc1Tg4FB35+RkRGUXRkAAgjkwg0MDMedGBh2AW3m+QtxCQuSgZxAl9h+gbAtvzEwJAN9VAXMxzc+Qlwa+JSBoRQUZL1AvBEggECB4wdMJqsYGH6zQ8IKlBWlgOF6/SowpoGGvQKaDgoqKSDxCWjAA2Cs6gF99AXIfgLEGsuA+kAJuwqYjRkBAgjk5S5gQQL00vHJDAwnLgFz1G9gPCElEbE/EMNAAGSBHjR4eIBiGtuBjKVQV4ABQACBDASG5t/dDAwWPQwMZkDbJlyApMG/UEN38UBc9hvIPwCMPFDy/AmM6UnXgN6eDywcniKHOEAAgQw8BsRBQGezASU7gfm9jYGh8hxQIzD2GIDZ7yYwjXwHuuYPMIHfApr2HxghbxcBY/giA4PmE6TUAgYAAQRilALxASCeBYwpb2A4bGBkTNnLAMmf2sDYBWbN73cZGDiAZeBxYBlp1w00CBg5DBlIXpUH4mcgBkAAMUDLw2So5CQQHxkDQRwQCzFAEn8oAyRVg/J+IVQM5ChgvmfYDFIPEEDIGudCDQ1HM9CFAZI9gckJXC2AggmUfwOh8mpQfZUgPkAAIWsEaToOxF+B2ATdpbgwEKRCDbQB8QECCF1SB4hBkXEeiPmJNBCUbEDZlwfEBwgg5CwBUnAFGDHpQCYw+TBsAbI3MUBqO2xFFyj9CELDdRlQLzg3AQQQLlujoAH9H6oRG/4HlT8ExCowvQABBgBOKHD8+UgEvgAAAABJRU5ErkJggg=="; +// tslint:disable-next-line: max-line-length +let touchIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S912AAAACXBIWXMAAAsTAAALEwEAmpwYAAAFEmlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgNS42LWMxNDIgNzkuMTYwOTI0LCAyMDE3LzA3LzEzLTAxOjA2OjM5ICAgICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cGhvdG9zaG9wPSJodHRwOi8vbnMuYWRvYmUuY29tL3Bob3Rvc2hvcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ0MgKFdpbmRvd3MpIiB4bXA6Q3JlYXRlRGF0ZT0iMjAxOS0wOS0xMVQwNjo1MzozMC0wNzowMCIgeG1wOk1vZGlmeURhdGU9IjIwMTktMDktMTFUMDY6NTQ6NDAtMDc6MDAiIHhtcDpNZXRhZGF0YURhdGU9IjIwMTktMDktMTFUMDY6NTQ6NDAtMDc6MDAiIGRjOmZvcm1hdD0iaW1hZ2UvcG5nIiBwaG90b3Nob3A6Q29sb3JNb2RlPSIzIiBwaG90b3Nob3A6SUNDUHJvZmlsZT0ic1JHQiBJRUM2MTk2Ni0yLjEiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6ODZmNTE5ZGQtN2E3MS1hOTQyLTgwNTktMjc3OTJjNTM1YTNlIiB4bXBNTTpEb2N1bWVudElEPSJ4bXAuZGlkOjg2ZjUxOWRkLTdhNzEtYTk0Mi04MDU5LTI3NzkyYzUzNWEzZSIgeG1wTU06T3JpZ2luYWxEb2N1bWVudElEPSJ4bXAuZGlkOjg2ZjUxOWRkLTdhNzEtYTk0Mi04MDU5LTI3NzkyYzUzNWEzZSI+IDx4bXBNTTpIaXN0b3J5PiA8cmRmOlNlcT4gPHJkZjpsaSBzdEV2dDphY3Rpb249ImNyZWF0ZWQiIHN0RXZ0Omluc3RhbmNlSUQ9InhtcC5paWQ6ODZmNTE5ZGQtN2E3MS1hOTQyLTgwNTktMjc3OTJjNTM1YTNlIiBzdEV2dDp3aGVuPSIyMDE5LTA5LTExVDA2OjUzOjMwLTA3OjAwIiBzdEV2dDpzb2Z0d2FyZUFnZW50PSJBZG9iZSBQaG90b3Nob3AgQ0MgKFdpbmRvd3MpIi8+IDwvcmRmOlNlcT4gPC94bXBNTTpIaXN0b3J5PiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PgqF3McAAAOgSURBVDiNlZVdbBRVFMd/d7o7penS7c6ghT60JRLCRx9s0JAIUvajJtCkVYw1IVaKUr/wpQk+qO8YRV8aEw3Q1EjpAwkNlkZNDMEYxUDoh9gSH7CNrXQptbtsHeluu7vHh5kuQ90l8k9ucuece373zJl77igRwa1YXK0BmoFdwFpAc1xZYBb4EegzAnKbPFJuYCyungEOAOuAsnwBwN/ADHDGCEhfQWAsrlqAl4DKAqCVmgV6jICcdhs1B/bUQ8IAHgH2x+IqeB8wFlcm9mvmYOFwq2kavsdNg22msWVzf39VSQFoBfBKLK787gx3A48tG44cCfpHhk/VwKYFn+9QFKL6wTZ9/Z07mvYfnK0aYIcbGARyGXzR/Wg5eOXq4Oj4H5Mnp3fVN0zBjZJjHz3tKwAscpLKATcAxcsGkUQRlGQMYzED0Ng4mgJNLEtXBYC6w8gBVxdY+DAqdwMXH7RSKQBFT090XXVVe+XQULl+7VqZPjWle1zLkssTD/Z5KuVeR+SRR2DWa1ndaxsingpIalCc9ftfu/XW4aH5UOhGIhK+l+Goe4eVsixdmeaB6Fxs5tdIw7MTUJ6urj40BaFEInG88oOjVzc1RJLNSqnmZeC5s2fXZ2u3tlfs3NG2Bma9duK2AoGF7OG3r1gAHR0X75pm0+zQ8MnbX39z5SYEE3V1rddBpoFP7OJAE3jPwFKx/ShA5eL4xK0xvz+bjUa9nlSqiJqaZDqR0LTBwQo9FIomLUtpY2OGtn37XK9p4AfeExGlgN+gqPjEiarzHm+m/mDbwmYQxidio35/NvuA75UFLgCfmgZvAu+LiNKAjZD5bt/zEx83NU32trRsG4GyTDqtKYDOzjpfIqFpS0uorq4tpZkMpFL8FQ63Xr982ew2AnLTvYsGXAL2mQa6EZAPP/v826P1u58cSqfVPLDQ/1VtkWV50um0Sg0M1GZEmIpGV305Mnzql7175v50nZbc5B3ge+C4UqpRRM719akL2P25dXJyVXB+Xv+9tDQ9c+knvWxw0Dy2d8/cXeANp+AA1cA0AM59+Krj7BQR3AN4GTCw2/MF7N4tATocmwcYB86LCO7ALgf64gpgBLs9vdi/BQ27f59z/BuduHdXAlcDPwP/AE+szLTQANod4M77gI6zFpgHhgH//wSexm5fn4jgbnBEZFQp9TrQCwwopfqd85bv6soAAaeuvSJi5T5Knl33O4UWJzDfyDr+H4ANy7H/AvM1z7/FQv/oAAAAAElFTkSuQmCC"; export function reset(): void { nodes = {}; @@ -120,8 +122,8 @@ export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { if (docElement === null) { let newDoc = doc.implementation.createHTMLDocument(""); docElement = newDoc.documentElement; - let pointer = doc.importNode(docElement, true); - doc.replaceChild(pointer, doc.documentElement); + let p = doc.importNode(docElement, true); + doc.replaceChild(p, doc.documentElement); if (doc.head) { doc.head.parentNode.removeChild(doc.head); } if (doc.body) { doc.body.parentNode.removeChild(doc.body); } } @@ -251,30 +253,38 @@ export function selection(data: ISelection, iframe: HTMLIFrameElement): void { s.setBaseAndExtent(element(data.start), data.startOffset, element(data.end), data.endOffset); } -export function mouse(event: Event, data: IMouse, iframe: HTMLIFrameElement): void { +export function pointer(event: Event, data: IPointer, iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; - let pointer = doc.getElementById("clarity-pointer"); + let p = doc.getElementById("clarity-pointer"); let pointerWidth = 20; let pointerHeight = 24; - if (pointer === null) { - pointer = doc.createElement("DIV"); - pointer.id = "clarity-pointer"; - pointer.style.position = "absolute"; - pointer.style.zIndex = "1000"; - pointer.style.width = pointerWidth + "px"; - pointer.style.height = pointerHeight + "px"; - doc.body.appendChild(pointer); + if (p === null) { + p = doc.createElement("DIV"); + p.id = "clarity-pointer"; + p.style.position = "absolute"; + p.style.zIndex = "1000"; + p.style.width = pointerWidth + "px"; + p.style.height = pointerHeight + "px"; + doc.body.appendChild(p); } - pointer.style.left = (data.x - 8) + "px"; - pointer.style.top = (data.y - 8) + "px"; + p.style.left = (data.x - 8) + "px"; + p.style.top = (data.y - 8) + "px"; switch (event) { case Event.Click: - pointer.style.background = `url(${clickIcon}) no-repeat left center`; + case Event.RightClick: + case Event.DoubleClick: + p.style.background = `url(${clickIcon}) no-repeat left center`; + break; + case Event.TouchMove: + case Event.TouchStart: + case Event.TouchEnd: + case Event.TouchCancel: + p.style.background = `url(${touchIcon}) no-repeat left center`; break; default: - pointer.style.background = `url(${pointerIcon}) no-repeat left center`; + p.style.background = `url(${pointerIcon}) no-repeat left center`; break; } } diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 9c46e19a..3ed302c2 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -4,7 +4,7 @@ import time from "@src/core/time"; import { queue } from "@src/data/upload"; import * as metric from "@src/metric"; import * as change from "./change"; -import * as mouse from "./mouse"; +import * as pointer from "./pointer"; import * as resize from "./resize"; import * as scroll from "./scroll"; import * as selection from "./selection"; @@ -22,16 +22,19 @@ export default function(type: Event): void { case Event.Click: case Event.DoubleClick: case Event.RightClick: - let m = mouse.data[type]; - for (let i = 0; i < m.length; i++) { - let entry = m[i]; + case Event.TouchStart: + case Event.TouchEnd: + case Event.TouchMove: + case Event.TouchCancel: + for (let i = 0; i < pointer.data[type].length; i++) { + let entry = pointer.data[type][i]; tokens = [entry.time, type]; tokens.push(entry.target); tokens.push(entry.x); tokens.push(entry.y); queue(tokens); } - mouse.reset(); + pointer.reset(); break; case Event.Resize: let r = resize.data; @@ -58,19 +61,18 @@ export default function(type: Event): void { change.reset(); break; case Event.Selection: - let sl = selection.data; - tokens.push(sl.start); - tokens.push(sl.startOffset); - tokens.push(sl.end); - tokens.push(sl.endOffset); + let s = selection.data; + tokens.push(s.start); + tokens.push(s.startOffset); + tokens.push(s.end); + tokens.push(s.endOffset); queue(tokens); metric.counter(Metric.Selections); selection.reset(); break; case Event.Scroll: - let s = scroll.data; - for (let i = 0; i < s.length; i++) { - let entry = s[i]; + for (let i = 0; i < scroll.data.length; i++) { + let entry = scroll.data[i]; tokens = [entry.time, type]; tokens.push(entry.target); tokens.push(entry.x); diff --git a/src/interaction/index.ts b/src/interaction/index.ts index 14327551..89ecab90 100644 --- a/src/interaction/index.ts +++ b/src/interaction/index.ts @@ -1,4 +1,4 @@ -import * as mouse from "@src/interaction/mouse"; +import * as pointer from "@src/interaction/pointer"; import * as resize from "@src/interaction/resize"; import * as scroll from "@src/interaction/scroll"; import * as selection from "@src/interaction/selection"; @@ -6,7 +6,7 @@ import * as unload from "@src/interaction/unload"; import * as visibility from "@src/interaction/visibility"; export function start(): void { - mouse.start(); + pointer.start(); resize.start(); visibility.start(); scroll.start(); diff --git a/src/interaction/mouse.ts b/src/interaction/mouse.ts deleted file mode 100644 index 68b45d55..00000000 --- a/src/interaction/mouse.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Event } from "@clarity-types/data"; -import { IMouse } from "@clarity-types/interaction"; -import config from "@src/core/config"; -import { bind } from "@src/core/event"; -import time from "@src/core/time"; -import { getId } from "@src/layout/dom"; -import encode from "./encode"; - -export let data: { [key: number]: IMouse[] } = {}; -let timeout: number = null; - -export function start(): void { - reset(); - bind(document, "mousedown", handler.bind(this, Event.MouseDown)); - bind(document, "mouseup", handler.bind(this, Event.MouseUp)); - bind(document, "mousemove", handler.bind(this, Event.MouseMove)); - bind(document, "mousewheel", handler.bind(this, Event.MouseWheel)); - bind(document, "dblclick", handler.bind(this, Event.DoubleClick)); - bind(document, "click", handler.bind(this, Event.Click)); -} - -function handler(event: Event, evt: MouseEvent): void { - let de = document.documentElement; - let x = "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null); - let y = "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null); - let current = {target: evt.target ? getId(evt.target as Node) : null, x, y, time: time()}; - switch (event) { - case Event.MouseMove: - case Event.MouseWheel: - let length = data[event].length; - let last = length > 1 ? data[event][length - 2] : null; - if (last && similar(last, current)) { data[event].pop(); } - data[event].push(current); - - if (timeout) { clearTimeout(timeout); } - timeout = window.setTimeout(encode, config.lookahead, event); - break; - case Event.Click: - event = evt.buttons === 2 || evt.button === 2 ? Event.RightClick : event; - default: - data[event].push(current); - encode(event); - break; - } -} - -export function reset(): void { - data = {}; - for (let event of [Event.MouseDown, Event.MouseUp, Event.MouseWheel, Event.MouseMove, Event.DoubleClick, Event.Click]) { - data[event] = []; - } -} - -function similar(last: IMouse, current: IMouse): boolean { - let dx = last.x - current.x; - let dy = last.y - current.y; - return (dx * dx + dy * dy < config.distance * config.distance) && (current.time - last.time < config.interval); -} diff --git a/src/interaction/pointer.ts b/src/interaction/pointer.ts new file mode 100644 index 00000000..e4d7985f --- /dev/null +++ b/src/interaction/pointer.ts @@ -0,0 +1,84 @@ +import { Event } from "@clarity-types/data"; +import { IPointer } from "@clarity-types/interaction"; +import config from "@src/core/config"; +import { bind } from "@src/core/event"; +import time from "@src/core/time"; +import { getId } from "@src/layout/dom"; +import encode from "./encode"; + +export let data: { [key: number]: IPointer[] } = {}; +let timeout: number = null; + +export function start(): void { + reset(); + bind(document, "mousedown", mouse.bind(this, Event.MouseDown)); + bind(document, "mouseup", mouse.bind(this, Event.MouseUp)); + bind(document, "mousemove", mouse.bind(this, Event.MouseMove)); + bind(document, "mousewheel", mouse.bind(this, Event.MouseWheel)); + bind(document, "dblclick", mouse.bind(this, Event.DoubleClick)); + bind(document, "click", mouse.bind(this, Event.Click)); + bind(document, "touchstart", touch.bind(this, Event.TouchStart)); + bind(document, "touchend", touch.bind(this, Event.TouchEnd)); + bind(document, "touchmove", touch.bind(this, Event.TouchMove)); + bind(document, "touchcancel", touch.bind(this, Event.TouchCancel)); +} + +function mouse(event: Event, evt: MouseEvent): void { + let de = document.documentElement; + let x = "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null); + let y = "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null); + let target = evt.target ? getId(evt.target as Node) : null; + event = event === Event.Click && (evt.buttons === 2 || evt.button === 2) ? Event.RightClick : event; + handler(event, {target, x, y, time: time()}); +} + +function touch(event: Event, evt: TouchEvent): void { + let de = document.documentElement; + let touches = evt.changedTouches; + let target = evt.target ? getId(evt.target as Node) : null; + if (touches) { + for (let i = 0; i < touches.length; i++) { + let t = touches[i]; + let x = "clientX" in t ? Math.round(t["clientX"] + de.scrollLeft) : null; + let y = "clientY" in t ? Math.round(t["clientY"] + de.scrollTop) : null; + handler(event, {target, x, y, time: time()}); + } + } +} + +function handler(event: Event, current: IPointer): void { + switch (event) { + case Event.MouseMove: + case Event.MouseWheel: + case Event.TouchMove: + let length = data[event].length; + let last = length > 1 ? data[event][length - 2] : null; + if (last && similar(last, current)) { data[event].pop(); } + data[event].push(current); + + if (timeout) { clearTimeout(timeout); } + timeout = window.setTimeout(encode, config.lookahead, event); + break; + default: + data[event].push(current); + encode(event); + break; + } +} + +export function reset(): void { + data = {}; + let mouseEvents = [Event.MouseDown, Event.MouseUp, Event.MouseWheel, Event.MouseMove, Event.DoubleClick, Event.Click]; + let touchEvents = [Event.TouchStart, Event.TouchMove, Event.TouchEnd, Event.TouchCancel]; + let events = mouseEvents.concat(touchEvents); + for (let event of events) { + data[event] = []; + } +} + +function similar(last: IPointer, current: IPointer): boolean { + let dx = last.x - current.x; + let dy = last.y - current.y; + let distance = Math.sqrt(dx * dx + dy * dy); + return (distance < config.distance) && (current.time - last.time < config.interval) && current.target === last.target; +} diff --git a/src/interaction/touch.ts b/src/interaction/touch.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index e5d6f010..fb876221 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -22,7 +22,7 @@ function schedule(): void { async function boxmodel(): Promise { let timer = Metric.BoxModelTime; task.start(timer); - let values = dom.getLeafNodes(); + let values = dom.boxmodel(); let doc = document.documentElement; let x = "pageXOffset" in window ? window.pageXOffset : doc.scrollLeft; let y = "pageYOffset" in window ? window.pageYOffset : doc.scrollTop; @@ -68,23 +68,6 @@ function update(id: number, box: number[]): void { } } -/* function getTextLayout(x: number, y: number, textNode: Node): number[] { - let layout: number[] = []; - let range = document.createRange(); - range.selectNodeContents(textNode); - let rects = range.getClientRects(); - - for (let i = 0; i < rects.length; i++) { - let rect = rects[i]; - layout.push(Math.floor(rect.left + x)); - layout.push(Math.floor(rect.top + y)); - layout.push(Math.floor(rect.width)); - layout.push(Math.floor(rect.height)); - } - - return layout.length > 0 ? layout : [0, 0, 0, 0]; -} */ - function getLayout(x: number, y: number, element: Element): number[] { let layout: number[] = [0, 0, 0, 0]; let rect = element.getBoundingClientRect(); diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 914c0ae0..053811b5 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -3,6 +3,7 @@ import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; const DEVTOOLS_HOOK: string = "__CLARITY_DEVTOOLS_HOOK__"; +const ID_ATTRIBUTE = "data-clarity"; const MASK_ATTRIBUTE = "data-clarity-mask"; const UNMASK_ATTRIBUTE = "data-clarity-unmask"; let index: number = 1; @@ -55,9 +56,9 @@ export function add(node: Node, data: INodeData, source: Source): void { children: [], data, selector: selector(id, data, parent ? parent.selector : ""), - metadata: { active: true, leaf: false, masked } + metadata: { active: true, layout: false, masked } }; - leaf(data.tag, id, parentId); + layout(data.tag, id, parentId); track(id, source); } @@ -106,7 +107,12 @@ export function update(node: Node, data: INodeData, source: Source): void { value["data"][key] = data[key]; } } - leaf(data.tag, id, parentId); + + // Update selector + let parent = parentId && parentId in values ? values[parentId] : null; + value.selector = selector(id, data, parent ? parent.selector : ""), + + layout(data.tag, id, parentId); track(id, source); } } @@ -134,10 +140,10 @@ export function has(node: Node): boolean { return getId(node) in nodes; } -export function getLeafNodes(): INodeValue[] { +export function boxmodel(): INodeValue[] { let v = []; for (let id in values) { - if (values[id].metadata.active && values[id].metadata.leaf) { + if (values[id].metadata.active && values[id].metadata.layout) { v.push(values[id]); } } @@ -186,6 +192,7 @@ function selector(id: number, data: INodeData, parent: string): string { case "LINK": case "META": case "*T": + case "*D": return ""; default: let value = getValue(id); @@ -193,12 +200,13 @@ function selector(id: number, data: INodeData, parent: string): string { let attributes = "attributes" in data ? data.attributes : {}; let current = "id" in attributes ? `${data.tag}#${attributes.id}` : `${parent}>${data.tag}`; if ("class" in attributes) { current = `${current}.${attributes.class.trim().split(" ").join(".")}`; } + if (ID_ATTRIBUTE in attributes) { current = `*${attributes[ID_ATTRIBUTE]}`; } if (current !== ex && selectorMap.indexOf(id) === -1) { selectorMap.push(id); } return current; } } -function leaf(tag: string, id: number, parentId: number): void { +function layout(tag: string, id: number, parentId: number): void { if (id !== null && parentId !== null) { switch (tag) { case "*T": @@ -209,7 +217,7 @@ function leaf(tag: string, id: number, parentId: number): void { for (let i = 0; i < value.length; i++) { let code = value.charCodeAt(i); if (!(code === 32 || code === 10 || code === 9 || code === 13)) { - values[parentId].metadata.leaf = true; + values[parentId].metadata.layout = true; break; } } @@ -218,7 +226,11 @@ function leaf(tag: string, id: number, parentId: number): void { case "IMG": case "IFRAME": case "svg:svg": - values[id].metadata.leaf = true; + values[id].metadata.layout = true; + break; + default: + // Capture layout for any element with a user defined selector + values[id].metadata.layout = values[id].selector.indexOf("*") === 0; break; } } diff --git a/types/data.d.ts b/types/data.d.ts index 0034cf54..d94f03ea 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -17,8 +17,10 @@ export const enum Event { MouseWheel, DoubleClick, RightClick, - Touch, - Keyboard, + TouchStart, + TouchEnd, + TouchMove, + TouchCancel, Selection, Resize, Scroll, diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 1d141410..fecae144 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -1,4 +1,4 @@ -export interface IMouse { +export interface IPointer { target: number; x: number; y: number; diff --git a/types/layout.d.ts b/types/layout.d.ts index 2665745f..3f5cd279 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -29,7 +29,7 @@ export interface INodeValue { export interface INodeMetadata { active: boolean; - leaf: boolean; + layout: boolean; masked: boolean; } From e4814fa6aff3908ea93e7d317948ff6648406f90 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 11 Sep 2019 08:28:03 -0700 Subject: [PATCH 093/105] 1.0.0-beta.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f431eeeb..5c31c1f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "0.4.0", + "version": "1.0.0-beta.0", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", From 65467d477b4edbdd3a2f79089ccfaec646d49868 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 11 Sep 2019 21:12:03 -0700 Subject: [PATCH 094/105] Simplifying decode logic --- decode/clarity.ts | 24 ++++++++++++------------ package.json | 2 +- src/core/version.ts | 2 +- src/data/upload.ts | 8 ++++---- types/data.d.ts | 11 ++++++++--- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index 7741e95c..1f781cf7 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,4 +1,4 @@ -import { Event, IDecodedEvent, IDecodedPayload, Token } from "../types/data"; +import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; import envelope from "./envelope"; import interaction from "./interaction"; import layout from "./layout"; @@ -9,10 +9,10 @@ import summarize from "./summary"; let pageId: string = null; -export function decode(data: string): IDecodedPayload { - let json = JSON.parse(data); +export function decode(data: string | IPayload): IDecodedPayload { + let json: IPayload = typeof data === "string" ? JSON.parse(data) : data; let time = Date.now(); - let payload: IDecodedPayload = { time, envelope: envelope(json.e), metrics: metric(json.m), analytics: [], playback: [], summary: [] }; + let payload: IDecodedPayload = { time, envelope: envelope(json.e), metrics: metric(json.m), stream: [], backup: [] }; let encoded: Token[][] = json.d; for (let entry of encoded) { @@ -36,30 +36,30 @@ export function decode(data: string): IDecodedPayload { case Event.TouchEnd: case Event.TouchMove: event = interaction(entry); - payload.analytics.push(event); + payload.stream.push(event); break; case Event.BoxModel: event = layout(entry); - payload.playback.push(event); + payload.backup.push(event); break; case Event.Discover: case Event.Mutation: event = layout(entry); summary = summarize(event); - payload.playback.push(event); - payload.summary.push(summary); + payload.stream.push(event); + payload.stream.push(summary); break; case Event.Checksum: event = layout(entry); - payload.summary.push(event); + payload.stream.push(event); break; case Event.Page: event = page(entry); - payload.analytics.push(event); + payload.backup.push(event); break; default: event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; - payload.playback.push(event); + payload.backup.push(event); break; } } @@ -83,7 +83,7 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head r.metric(decoded.metrics, header); // Replay events - let events = [...decoded.analytics, ...decoded.playback, ...decoded.summary].sort(sort); + let events = [...decoded.stream, ...decoded.backup].sort(sort); replay(events, iframe); } diff --git a/package.json b/package.json index 5c31c1f3..dfba4f52 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.0", + "version": "1.0.0-beta.1", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/core/version.ts b/src/core/version.ts index 77e61c40..24e7307b 100644 --- a/src/core/version.ts +++ b/src/core/version.ts @@ -1,2 +1,2 @@ -let version = "1.0"; +let version = "1.0.0"; export default version; diff --git a/src/data/upload.ts b/src/data/upload.ts index d24ebfe1..20dca96b 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,4 +1,4 @@ -import { Event, IPayload, Token } from "@clarity-types/data"; +import { Event, ISerializedPayload, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; @@ -50,13 +50,13 @@ export function end(): void { function upload(last: boolean = false): void { let handler = config.upload ? config.upload : send; - let payload: IPayload = {e: JSON.stringify(envelope(last)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; + let payload: ISerializedPayload = {e: JSON.stringify(envelope(last)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; handler(stringify(payload), last); backup(payload); events = []; } -function stringify(payload: IPayload): string { +function stringify(payload: ISerializedPayload): string { return `{"e":${payload.e},"m":${payload.m},"d":${payload.d}}`; } @@ -82,7 +82,7 @@ function recover(): void { } } -function backup(payload: IPayload): void { +function backup(payload: ISerializedPayload): void { if ("localStorage" in window) { payload.e = JSON.stringify(envelope(true, true)); localStorage.setItem("clarity-backup", stringify(payload)); diff --git a/types/data.d.ts b/types/data.d.ts index d94f03ea..69377431 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -41,6 +41,12 @@ export const enum Upload { } export interface IPayload { + e: Token[]; + m: Token[]; + d: Token[][]; +} + +export interface ISerializedPayload { e: string; m: string; d: string; @@ -50,9 +56,8 @@ export interface IDecodedPayload { time: number; envelope: IEnvelope; metrics: IDecodedMetric; - analytics: IDecodedEvent[]; - playback: IDecodedEvent[]; - summary: IDecodedEvent[]; + stream: IDecodedEvent[]; + backup: IDecodedEvent[]; } export interface IDecodedEvent { From 99cb6e270d2fffe57f0ae9079ffa9962bbe1807d Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 13 Sep 2019 08:48:41 -0700 Subject: [PATCH 095/105] Adding support for ping & pause / restart --- decode/envelope.ts | 4 ++-- src/clarity.ts | 29 ++++++++++++++++++++++++++--- src/core/config.ts | 6 ++++-- src/core/index.ts | 17 +++++++++-------- src/core/task.ts | 2 +- src/core/time.ts | 4 +++- src/data/encode.ts | 23 ++++++++++++++++------- src/data/index.ts | 4 ++++ src/data/metadata.ts | 35 +++++++++++++++++++++++++---------- src/data/ping.ts | 37 +++++++++++++++++++++++++++++++++++++ src/data/upload.ts | 3 +++ types/core.d.ts | 8 +++++--- types/data.d.ts | 15 +++++++++++++-- types/index.d.ts | 5 ++++- 14 files changed, 152 insertions(+), 40 deletions(-) create mode 100644 src/data/ping.ts diff --git a/decode/envelope.ts b/decode/envelope.ts index 53373f09..af3a2906 100644 --- a/decode/envelope.ts +++ b/decode/envelope.ts @@ -4,10 +4,10 @@ export default function(tokens: Token[]): IEnvelope { return { sequence: tokens[0] as number, version: tokens[1] as string, - pageId: tokens[2] as string, + projectId: tokens[2] as string, userId: tokens[3] as string, sessionId: tokens[4] as string, - projectId: tokens[5] as string, + pageId: tokens[5] as string, upload: tokens[6] as Upload, end: tokens[7] as number }; diff --git a/src/clarity.ts b/src/clarity.ts index daa8bb8a..3b87d89d 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,5 +1,7 @@ import { IConfig } from "@clarity-types/core"; import * as core from "@src/core"; +import configuration from "@src/core/config"; +import { bind } from "@src/core/event"; import * as data from "@src/data"; import * as diagnostic from "@src/diagnostic"; import * as interaction from "@src/interaction"; @@ -8,10 +10,19 @@ import * as metric from "@src/metric"; let status = false; -/* Initial discovery of DOM */ -export function start(configuration: IConfig = {}): void { +export function config(override: IConfig): boolean { + // Process custom configuration overrides, if available + if (status) { return false; } + for (let key in override) { + if (key in configuration) { configuration[key] = override[key]; } + } + return true; +} + +export function start(override: IConfig = {}): void { if (core.check()) { - core.start(configuration); + config(override); + core.start(); metric.start(); data.start(); diagnostic.start(); @@ -23,6 +34,18 @@ export function start(configuration: IConfig = {}): void { } } +export function pause(): void { + end(); + bind(document, "mousemove", resume); + bind(document, "touchstart", resume); + bind(window, "resize", resume); + bind(window, "scroll", resume); +} + +export function resume(): void { + start(); +} + export function end(): void { if (status) { interaction.end(); diff --git a/src/core/config.ts b/src/core/config.ts index d6aa5e56..6d2811df 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,18 +1,20 @@ import { IConfig } from "@clarity-types/core"; let config: IConfig = { - sessionId: null, projectId: null, - longTask: 30, + longtask: 30, lookahead: 500, distance: 20, interval: 25, delay: 1000, expire: 7, + ping: 60 * 1000, + timeout: 10 * 60 * 1000, cssRules: false, lean: false, tokens: [], url: "", + onstart: null, upload: null }; diff --git a/src/core/index.ts b/src/core/index.ts index d9eba4e7..9174cd2e 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -1,20 +1,21 @@ -import { IConfig } from "@clarity-types/core"; -import config from "@src/core/config"; import * as event from "@src/core/event"; -export function start(configuration: IConfig): void { - // Process custom configuration, if available - for (let key in configuration) { - if (key in config) { config[key] = configuration[key]; } - } +export let startTime = 0; +export function start(): void { + startTime = performance.now(); event.reset(); } export function end(): void { event.reset(); + startTime = 0; } export function check(): boolean { - return window["MutationObserver"] && document["createTreeWalker"] && "now" in Date ? true : false; + try { + return window["MutationObserver"] && document["createTreeWalker"] && "now" in Date && "now" in performance ? true : false; + } catch (ex) { + return false; + } } diff --git a/src/core/task.ts b/src/core/task.ts index c11457d8..242ac54c 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -4,7 +4,7 @@ import config from "@src/core/config"; import * as metrics from "@src/metric"; let tracker: ITaskTracker = {}; -let threshold = config.longTask; +let threshold = config.longtask; let queue: IAsyncTask[] = []; let active: IAsyncTask = null; diff --git a/src/core/time.ts b/src/core/time.ts index 38e48f6e..55d352d9 100644 --- a/src/core/time.ts +++ b/src/core/time.ts @@ -1,3 +1,5 @@ +import { startTime } from "@src/core"; + export default function(): number { - return Math.round(performance.now()); + return Math.round(performance.now() - startTime); } diff --git a/src/data/encode.ts b/src/data/encode.ts index b7951ddd..d2a1dc56 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -2,15 +2,24 @@ import {Event, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import time from "@src/core/time"; import { metadata } from "@src/data/metadata"; +import { data as pingdata } from "@src/data/ping"; import * as metric from "@src/metric"; import { queue } from "./upload"; -export default function(): void { +export default function(event: Event): void { let t = time(); - let tokens: Token[] = [t, Event.Page]; - metric.counter(Metric.StartTime, t); - tokens.push(metadata.page.url); - tokens.push(metadata.page.title); - tokens.push(metadata.page.referrer); - queue(tokens); + let tokens: Token[] = [t, event]; + switch (event) { + case Event.Ping: + tokens.push(pingdata.gap); + queue(tokens); + break; + case Event.Page: + metric.counter(Metric.StartTime, Math.round(performance.now())); + tokens.push(metadata.page.url); + tokens.push(metadata.page.title); + tokens.push(metadata.page.referrer); + queue(tokens); + break; + } } diff --git a/src/data/index.ts b/src/data/index.ts index b43ed522..7a6b8ba0 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -1,11 +1,15 @@ import * as metadata from "@src/data/metadata"; +import * as ping from "@src/data/ping"; import * as upload from "@src/data/upload"; + export function start(): void { upload.start(); metadata.start(); + ping.start(); } export function end(): void { + ping.end(); upload.end(); metadata.end(); } diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 496d6a3c..7fcd6163 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,4 +1,4 @@ -import { ICookieData, IEnvelope, IMetadata, IPage, Token, Upload } from "@clarity-types/data"; +import { Event, ICookieData, IEnvelope, IMetadata, IPage, Token, Upload } from "@clarity-types/data"; import config from "@src/core/config"; import version from "@src/core/version"; import encode from "@src/data/encode"; @@ -11,16 +11,17 @@ export let metadata: IMetadata = null; export function start(): void { let cookie: ICookieData = read(); - let timestamp = Date.now(); - let pageId = id(); + let ts = Date.now(); let projectId = config.projectId || hash(location.host); - let userId = cookie && cookie.userId ? cookie.userId : id(); - let sessionId = cookie && cookie.sessionId && timestamp - cookie.timestamp < CLARITY_SESSION_LENGTH ? cookie.sessionId : id(); + let userId = cookie && cookie.userId ? cookie.userId : guid(); + let sessionId = cookie && cookie.sessionId && ts - cookie.timestamp < CLARITY_SESSION_LENGTH ? cookie.sessionId : ts.toString(36); + let pageId = guid(); let e: IEnvelope = { sequence: 0, version, pageId, userId, sessionId, projectId, upload: Upload.Async, end: 0 }; let p: IPage = { url: location.href, title: document.title, referrer: document.referrer }; metadata = { page: p, envelope: e }; - track({ userId, sessionId, timestamp }); - encode(); + track({ userId, sessionId, timestamp: ts }); + encode(Event.Page); + if (config.onstart) { config.onstart({ userId, sessionId, pageId}); } } export function end(): void { @@ -31,12 +32,26 @@ export function envelope(last: boolean, backup: boolean = false): Token[] { let upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); let e = metadata.envelope; if (upload !== Upload.Backup) { e.sequence++; } - return [e.sequence, e.version, e.pageId, e.userId, e.sessionId, e.projectId, upload, last ? 1 : 0]; + return [e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, upload, last ? 1 : 0]; } -function id(): string { - return Math.random().toString(36).slice(-6) + Date.now().toString(36).slice(-4); +// Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript +// Excluding 3rd party code from tslint +// tslint:disable +function guid() { + let d = new Date().getTime(); + if (window.performance && performance.now) { + // Use high-precision timer if available + d += performance.now(); + } + let uuid = "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, function(c) { + let r = (d + Math.random() * 16) % 16 | 0; + d = Math.floor(d / 16); + return (c == "x" ? r : (r & 0x3 | 0x8)).toString(16); + }); + return uuid; } +// tslint:enable function track(data: ICookieData): void { let expiry = new Date(); diff --git a/src/data/ping.ts b/src/data/ping.ts new file mode 100644 index 00000000..80e579cf --- /dev/null +++ b/src/data/ping.ts @@ -0,0 +1,37 @@ +import { Event, IPing } from "@clarity-types/data"; +import { pause } from "@src/clarity"; +import config from "@src/core/config"; +import time from "@src/core/time"; +import encode from "./encode"; + +export let data: IPing; +let last = 0; +let interval = 0; +let timeout: number = null; + +export function start(): void { + interval = config.ping; +} + +export function reset(): void { + if (timeout) { clearTimeout(timeout); } + timeout = window.setTimeout(ping, interval); +} + +function ping(): void { + let now = time(); + data = { gap: now - last }; + encode(Event.Ping); + if (data.gap < config.timeout) { + interval = Math.min(interval * 2, config.timeout); + timeout = window.setTimeout(ping, interval); + } else { pause(); } + + last = now; +} + +export function end(): void { + if (timeout) { clearTimeout(timeout); } + last = 0; + interval = 0; +} diff --git a/src/data/upload.ts b/src/data/upload.ts index 20dca96b..ba06ce50 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -2,6 +2,7 @@ import { Event, ISerializedPayload, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; +import * as ping from "@src/data/ping"; import { counter } from "@src/metric"; import metrics from "@src/metric/encode"; @@ -44,6 +45,7 @@ export function queue(data: Token[]): void { } export function end(): void { + clearTimeout(timeout); upload(true); events = []; } @@ -54,6 +56,7 @@ function upload(last: boolean = false): void { handler(stringify(payload), last); backup(payload); events = []; + if (!last) { ping.reset(); } } function stringify(payload: ISerializedPayload): string { diff --git a/types/core.d.ts b/types/core.d.ts index ba69450c..764d6e2d 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -1,4 +1,4 @@ -import { IPayload, Token } from "./data"; +import { IClarityData, IPayload, Token } from "./data"; type TaskFunction = () => Promise; type TaskResolve = () => void; @@ -13,18 +13,20 @@ export interface IBindingContainer { } export interface IConfig { - sessionId?: string; projectId?: string; - longTask?: number; + longtask?: number; lookahead?: number; distance?: number; interval?: number; delay?: number; expire?: number; + ping?: number; + timeout?: number; cssRules?: boolean; lean?: boolean; tokens?: string[]; url?: string; + onstart?: (data: IClarityData) => void; upload?: (data: string, last: boolean) => void; } diff --git a/types/data.d.ts b/types/data.d.ts index 69377431..eb9a90b2 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -10,6 +10,7 @@ export const enum Event { Mutation, BoxModel, Checksum, + Ping, Click, MouseMove, MouseDown, @@ -72,6 +73,12 @@ export interface ICookieData { timestamp: number; } +export interface IClarityData { + userId: string; + sessionId: string; + pageId: string; +} + export interface IMetadata { page: IPage; envelope: IEnvelope; @@ -86,10 +93,14 @@ export interface IPage { export interface IEnvelope { sequence: number; version: string; - pageId: string; + projectId: string; userId: string; sessionId: string; - projectId: string; + pageId: string; upload: Upload; end: number; } + +export interface IPing { + gap: number; +} diff --git a/types/index.d.ts b/types/index.d.ts index d910d825..ef6124e1 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -2,7 +2,10 @@ import { IConfig } from "./core"; interface IClarityJs { version: string; - start: (config: IConfig) => void; + config: (config?: IConfig) => boolean; + start: (config?: IConfig) => void; + pause: () => void; + resume: () => void; end: () => void; active: () => boolean; } From 9ebfea94c6918e50d67475d999d20a4c0e367385 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 13 Sep 2019 11:44:44 -0700 Subject: [PATCH 096/105] Changing package exports --- .gitignore | 2 ++ package.json | 6 +++--- webpack/configs/index.ts | 5 +++-- webpack/configs/prod.ts | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 16bc9692..e7c06c85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ # Build results build/ +clarity.js +decode.js # npm packages .rpt2_cache diff --git a/package.json b/package.json index dfba4f52..e87d9fe6 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "clarity-js", - "version": "1.0.0-beta.1", + "version": "1.0.0-beta.2", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", - "main": "build/index.js", + "main": "clarity.js", "types": "types/index.d.ts", "keywords": [ "clarity", @@ -66,7 +66,7 @@ "build:main": "webpack --config webpack/configs/index.ts", "build:cjs": "webpack --config webpack/configs/prod.ts", "build:cjs:dev": "webpack --config webpack/configs/dev.ts", - "build:clean": "del-cli build/*", + "build:clean": "del-cli build/* && del-cli clarity.js && del-cli decode.js", "tslint": "tslint --project ./", "tslint:fix": "tslint --fix --project ./ --force" }, diff --git a/webpack/configs/index.ts b/webpack/configs/index.ts index de8f26ad..13c824cc 100644 --- a/webpack/configs/index.ts +++ b/webpack/configs/index.ts @@ -10,13 +10,14 @@ const IndexConfig: webpack.Configuration = { mode: "production", entry: { - index: "./src/index.ts", + clarity: "./src/index.ts", decode: "./decode/clarity.ts" }, output: { libraryTarget: "commonjs", - filename: "[name].js" + filename: "[name].js", + path: `${__dirname}/../../` }, optimization: { diff --git a/webpack/configs/prod.ts b/webpack/configs/prod.ts index 999d425b..4cb87fe4 100644 --- a/webpack/configs/prod.ts +++ b/webpack/configs/prod.ts @@ -16,6 +16,7 @@ const ProdConfig: webpack.Configuration = { }, output: { + path: `${__dirname}/../../build`, filename: "[name].min.js" }, From f805562a9232de70be45ef797130c2e858edcddf Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 15 Sep 2019 12:43:09 -0700 Subject: [PATCH 097/105] Supporting pagehide and checking for promise --- package.json | 3 ++- src/clarity.ts | 1 + src/core/index.ts | 2 +- src/data/upload.ts | 7 +++---- src/interaction/unload.ts | 1 + src/interaction/visibility.ts | 2 -- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index e87d9fe6..c84dcf2d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.2", + "version": "1.0.0-beta.3", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", @@ -63,6 +63,7 @@ }, "scripts": { "build": "yarn build:clean && yarn build:main && yarn build:cjs", + "build:dev": "yarn build:clean && yarn build:main && yarn build:cjs:dev", "build:main": "webpack --config webpack/configs/index.ts", "build:cjs": "webpack --config webpack/configs/prod.ts", "build:cjs:dev": "webpack --config webpack/configs/dev.ts", diff --git a/src/clarity.ts b/src/clarity.ts index 3b87d89d..a9570c86 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -40,6 +40,7 @@ export function pause(): void { bind(document, "touchstart", resume); bind(window, "resize", resume); bind(window, "scroll", resume); + bind(window, "pageshow", resume); } export function resume(): void { diff --git a/src/core/index.ts b/src/core/index.ts index 9174cd2e..896bd00b 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -14,7 +14,7 @@ export function end(): void { export function check(): boolean { try { - return window["MutationObserver"] && document["createTreeWalker"] && "now" in Date && "now" in performance ? true : false; + return typeof Promise !== "undefined" && MutationObserver && document["createTreeWalker"] && "now" in Date && "now" in performance; } catch (ex) { return false; } diff --git a/src/data/upload.ts b/src/data/upload.ts index ba06ce50..b8956fad 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -54,7 +54,7 @@ function upload(last: boolean = false): void { let handler = config.upload ? config.upload : send; let payload: ISerializedPayload = {e: JSON.stringify(envelope(last)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; handler(stringify(payload), last); - backup(payload); + if (last) { backup(payload); } events = []; if (!last) { ping.reset(); } } @@ -70,14 +70,13 @@ function send(data: string, last: boolean = false): void { } else { let xhr = new XMLHttpRequest(); xhr.open("POST", config.url); - xhr.setRequestHeader("Content-Type", "application/json"); xhr.send(data); } } } function recover(): void { - if ("localStorage" in window) { + if (typeof localStorage !== "undefined") { let data = localStorage.getItem("clarity-backup"); if (data && data.length > 0) { send(data); @@ -86,7 +85,7 @@ function recover(): void { } function backup(payload: ISerializedPayload): void { - if ("localStorage" in window) { + if (typeof localStorage !== "undefined") { payload.e = JSON.stringify(envelope(true, true)); localStorage.setItem("clarity-backup", stringify(payload)); } diff --git a/src/interaction/unload.ts b/src/interaction/unload.ts index e6b8688d..846af2c4 100644 --- a/src/interaction/unload.ts +++ b/src/interaction/unload.ts @@ -9,6 +9,7 @@ export let data: IUnload; export function start(): void { bind(window, "beforeunload", recompute); bind(window, "unload", recompute); + bind(window, "pagehide", recompute); } function recompute(evt: UIEvent): void { diff --git a/src/interaction/visibility.ts b/src/interaction/visibility.ts index 2b96edda..f27b4a50 100644 --- a/src/interaction/visibility.ts +++ b/src/interaction/visibility.ts @@ -6,8 +6,6 @@ import encode from "./encode"; export let data: IVisibility; export function start(): void { - bind(window, "pagehide", recompute); - bind(window, "pageshow", recompute); bind(document, "visibilitychange", recompute); recompute(); } From 148aec715149bacfb205badfd59e003e130485e8 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Wed, 18 Sep 2019 09:13:25 -0700 Subject: [PATCH 098/105] Clearing timeouts while ending Clarity session --- decode/clarity.ts | 17 +++++--- decode/diagnostic.ts | 24 ++++++++++++ decode/envelope.ts | 17 ++++---- decode/page.ts | 13 +++---- package.json | 2 +- src/clarity.ts | 5 +-- src/core/event.ts | 26 ++++--------- src/data/encode.ts | 2 + src/data/metadata.ts | 15 ++++++-- src/data/ping.ts | 2 +- src/diagnostic/image.ts | 4 -- src/diagnostic/index.ts | 4 +- src/diagnostic/script.ts | 4 -- src/interaction/index.ts | 4 +- src/interaction/pointer.ts | 7 +++- src/interaction/scroll.ts | 7 +++- src/interaction/selection.ts | 9 ++++- src/layout/boxmodel.ts | 9 ++++- src/layout/dom.ts | 1 + src/layout/index.ts | 3 ++ src/layout/mutation.ts | 1 + src/layout/node.ts | 2 +- types/core.d.ts | 8 ++-- types/data.d.ts | 75 ++++++++++++++++++++---------------- types/metric.d.ts | 46 +++++++++++----------- 25 files changed, 180 insertions(+), 127 deletions(-) create mode 100644 decode/diagnostic.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 1f781cf7..7c78593a 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,4 +1,5 @@ -import { Event, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; +import { Event, IAugmentation, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; +import diagnostic from "./diagnostic"; import envelope from "./envelope"; import interaction from "./interaction"; import layout from "./layout"; @@ -9,10 +10,11 @@ import summarize from "./summary"; let pageId: string = null; -export function decode(data: string | IPayload): IDecodedPayload { +export function decode(data: string | IPayload, augmentations: IAugmentation = null): IDecodedPayload { let json: IPayload = typeof data === "string" ? JSON.parse(data) : data; - let time = Date.now(); - let payload: IDecodedPayload = { time, envelope: envelope(json.e), metrics: metric(json.m), stream: [], backup: [] }; + let timestamp = augmentations ? augmentations.timestamp : Date.now(); + let ua = augmentations ? augmentations.ua : (navigator && "userAgent" in navigator ? navigator.userAgent : ""); + let payload: IDecodedPayload = { timestamp, ua, envelope: envelope(json.e), metrics: metric(json.m), stream: [], backup: [] }; let encoded: Token[][] = json.d; for (let entry of encoded) { @@ -55,7 +57,12 @@ export function decode(data: string | IPayload): IDecodedPayload { break; case Event.Page: event = page(entry); - payload.backup.push(event); + payload.stream.push(event); + break; + case Event.ScriptError: + case Event.ImageError: + event = diagnostic(entry); + payload.stream.push(event); break; default: event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; diff --git a/decode/diagnostic.ts b/decode/diagnostic.ts new file mode 100644 index 00000000..d26c9eeb --- /dev/null +++ b/decode/diagnostic.ts @@ -0,0 +1,24 @@ +import { Event, IDecodedEvent, Token } from "../types/data"; +import { IImageError, IScriptError } from "../types/diagnostic"; + +export default function(tokens: Token[]): IDecodedEvent { + let time = tokens[0] as number; + let event = tokens[1] as Event; + switch (event) { + case Event.ImageError: + let imageError: IImageError = { + source: tokens[2] as string, + target: tokens[3] as number + }; + return { time, event, data: imageError }; + case Event.Selection: + let scriptError: IScriptError = { + source: tokens[2] as string, + message: tokens[3] as string, + line: tokens[4] as number, + column: tokens[5] as number, + stack: tokens[6] as string + }; + return { time, event, data: scriptError }; + } +} diff --git a/decode/envelope.ts b/decode/envelope.ts index af3a2906..ab21dc79 100644 --- a/decode/envelope.ts +++ b/decode/envelope.ts @@ -2,13 +2,14 @@ import { IEnvelope, Token, Upload } from "../types/data"; export default function(tokens: Token[]): IEnvelope { return { - sequence: tokens[0] as number, - version: tokens[1] as string, - projectId: tokens[2] as string, - userId: tokens[3] as string, - sessionId: tokens[4] as string, - pageId: tokens[5] as string, - upload: tokens[6] as Upload, - end: tokens[7] as number + elapsed: tokens[0] as number, + sequence: tokens[1] as number, + version: tokens[2] as string, + projectId: tokens[3] as string, + userId: tokens[4] as string, + sessionId: tokens[5] as string, + pageId: tokens[6] as string, + upload: tokens[7] as Upload, + end: tokens[8] as number }; } diff --git a/decode/page.ts b/decode/page.ts index 20f07af7..83d34e0e 100644 --- a/decode/page.ts +++ b/decode/page.ts @@ -5,14 +5,11 @@ export default function(tokens: Token[]): IDecodedEvent { time: tokens[0] as number, event: tokens[1] as Event, data: { - sequence: tokens[2] as number, - version: tokens[3] as string, - pageId: tokens[4] as string, - userId: tokens[5] as string, - projectId: tokens[6] as string, - url: tokens[7] as string, - title: tokens[8] as string, - referrer: tokens[9] as string + timestamp: tokens[2] as number, + elapsed: tokens[3] as number, + url: tokens[4] as string, + title: tokens[5] as string, + referrer: tokens[6] as string } }; } diff --git a/package.json b/package.json index c84dcf2d..d3b5ab58 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.3", + "version": "1.0.0-beta.4", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/clarity.ts b/src/clarity.ts index a9570c86..07384102 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -22,15 +22,14 @@ export function config(override: IConfig): boolean { export function start(override: IConfig = {}): void { if (core.check()) { config(override); + status = true; + core.start(); metric.start(); data.start(); diagnostic.start(); dom.start(); interaction.start(); - - // Mark Clarity session as active - status = true; } } diff --git a/src/core/event.ts b/src/core/event.ts index 9a37c1a7..769803ef 100644 --- a/src/core/event.ts +++ b/src/core/event.ts @@ -1,26 +1,16 @@ -import { IBindingContainer, IEventBindingPair } from "@clarity-types/core"; +import { IEventBindingData } from "@clarity-types/core"; -let bindings: IBindingContainer = {}; +let bindings: IEventBindingData[] = []; -export function bind(target: EventTarget, event: string, listener: EventListener, useCapture: boolean = false): void { - let eventBindings = bindings[event] || []; - target.addEventListener(event, listener, useCapture); - eventBindings.push({ - target, - listener - }); - bindings[event] = eventBindings; +export function bind(target: EventTarget, event: string, listener: EventListener, capture: boolean = false): void { + target.addEventListener(event, listener, capture); + bindings.push({ event, target, listener, capture }); } export function reset(): void { // Walk through existing list of bindings and remove them all - for (let evt in bindings) { - if (bindings.hasOwnProperty(evt)) { - let eventBindings = bindings[evt] as IEventBindingPair[]; - for (let i = 0; i < eventBindings.length; i++) { - (eventBindings[i].target).removeEventListener(evt, eventBindings[i].listener); - } - } + for (let binding of bindings) { + (binding.target).removeEventListener(binding.event, binding.listener, binding.capture); } - bindings = {}; + bindings = []; } diff --git a/src/data/encode.ts b/src/data/encode.ts index d2a1dc56..00a8cc28 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -16,6 +16,8 @@ export default function(event: Event): void { break; case Event.Page: metric.counter(Metric.StartTime, Math.round(performance.now())); + tokens.push(metadata.page.timestamp); + tokens.push(metadata.page.elapsed); tokens.push(metadata.page.url); tokens.push(metadata.page.title); tokens.push(metadata.page.referrer); diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 7fcd6163..5294ec56 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,5 +1,6 @@ import { Event, ICookieData, IEnvelope, IMetadata, IPage, Token, Upload } from "@clarity-types/data"; import config from "@src/core/config"; +import time from "@src/core/time"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; @@ -16,9 +17,11 @@ export function start(): void { let userId = cookie && cookie.userId ? cookie.userId : guid(); let sessionId = cookie && cookie.sessionId && ts - cookie.timestamp < CLARITY_SESSION_LENGTH ? cookie.sessionId : ts.toString(36); let pageId = guid(); - let e: IEnvelope = { sequence: 0, version, pageId, userId, sessionId, projectId, upload: Upload.Async, end: 0 }; - let p: IPage = { url: location.href, title: document.title, referrer: document.referrer }; + let e: IEnvelope = { elapsed: time(), sequence: 0, version, pageId, userId, sessionId, projectId, upload: Upload.Async, end: 0 }; + let p: IPage = { timestamp: ts, elapsed: time(), url: location.href, title: document.title, referrer: document.referrer }; + metadata = { page: p, envelope: e }; + track({ userId, sessionId, timestamp: ts }); encode(Event.Page); if (config.onstart) { config.onstart({ userId, sessionId, pageId}); } @@ -31,8 +34,12 @@ export function end(): void { export function envelope(last: boolean, backup: boolean = false): Token[] { let upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); let e = metadata.envelope; - if (upload !== Upload.Backup) { e.sequence++; } - return [e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, upload, last ? 1 : 0]; + if (upload !== Upload.Backup) { + e.elapsed = time(); + e.sequence++; + } + + return [e.elapsed, e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, upload, last ? 1 : 0]; } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript diff --git a/src/data/ping.ts b/src/data/ping.ts index 80e579cf..97054144 100644 --- a/src/data/ping.ts +++ b/src/data/ping.ts @@ -31,7 +31,7 @@ function ping(): void { } export function end(): void { - if (timeout) { clearTimeout(timeout); } + clearTimeout(timeout); last = 0; interval = 0; } diff --git a/src/diagnostic/image.ts b/src/diagnostic/image.ts index f893bd89..64753320 100644 --- a/src/diagnostic/image.ts +++ b/src/diagnostic/image.ts @@ -10,10 +10,6 @@ export function start(): void { bind(document, "error", handler, true); } -export function end(): void { - return; -} - function handler(error: ErrorEvent): void { let target = error.target as HTMLElement; if (target && target.tagName === "IMG") { diff --git a/src/diagnostic/index.ts b/src/diagnostic/index.ts index d3cd1def..760d583a 100644 --- a/src/diagnostic/index.ts +++ b/src/diagnostic/index.ts @@ -7,6 +7,6 @@ export function start(): void { } export function end(): void { - script.end(); - image.end(); + script.reset(); + image.reset(); } diff --git a/src/diagnostic/script.ts b/src/diagnostic/script.ts index ad1f6cf0..69e13f1a 100644 --- a/src/diagnostic/script.ts +++ b/src/diagnostic/script.ts @@ -9,10 +9,6 @@ export function start(): void { bind(window, "error", handler); } -export function end(): void { - return; -} - function handler(error: ErrorEvent): void { let e = error["error"] || error; diff --git a/src/interaction/index.ts b/src/interaction/index.ts index 89ecab90..e515c6cd 100644 --- a/src/interaction/index.ts +++ b/src/interaction/index.ts @@ -15,5 +15,7 @@ export function start(): void { } export function end(): void { - // End calls + pointer.end(); + scroll.end(); + selection.end(); } diff --git a/src/interaction/pointer.ts b/src/interaction/pointer.ts index e4d7985f..b089bd76 100644 --- a/src/interaction/pointer.ts +++ b/src/interaction/pointer.ts @@ -56,7 +56,7 @@ function handler(event: Event, current: IPointer): void { if (last && similar(last, current)) { data[event].pop(); } data[event].push(current); - if (timeout) { clearTimeout(timeout); } + clearTimeout(timeout); timeout = window.setTimeout(encode, config.lookahead, event); break; default: @@ -82,3 +82,8 @@ function similar(last: IPointer, current: IPointer): boolean { let distance = Math.sqrt(dx * dx + dy * dy); return (distance < config.distance) && (current.time - last.time < config.interval) && current.target === last.target; } + +export function end(): void { + clearTimeout(timeout); + data = {}; +} diff --git a/src/interaction/scroll.ts b/src/interaction/scroll.ts index fc5eebe1..185f0230 100644 --- a/src/interaction/scroll.ts +++ b/src/interaction/scroll.ts @@ -25,7 +25,7 @@ function recompute(event: UIEvent = null): void { if (last && similar(last, current)) { data.pop(); } data.push(current); - if (timeout) { clearTimeout(timeout); } + clearTimeout(timeout); timeout = window.setTimeout(encode, config.lookahead, Event.Scroll); } @@ -38,3 +38,8 @@ function similar(last: IScroll, current: IScroll): boolean { let dy = last.y - current.y; return (dx * dx + dy * dy < config.distance * config.distance) && (current.time - last.time < config.interval); } + +export function end(): void { + clearTimeout(timeout); + data = []; +} diff --git a/src/interaction/selection.ts b/src/interaction/selection.ts index 2183fbd3..25372b3d 100644 --- a/src/interaction/selection.ts +++ b/src/interaction/selection.ts @@ -19,7 +19,7 @@ function recompute(): void { let s = document.getSelection(); if (selection !== null && data.start !== null && data.start !== getId(s.anchorNode)) { - if (timeout) { clearTimeout(timeout); } + clearTimeout(timeout); encode(Event.Selection); } @@ -31,7 +31,7 @@ function recompute(): void { }; selection = s; - if (timeout) { clearTimeout(timeout); } + clearTimeout(timeout); timeout = window.setTimeout(encode, config.lookahead, Event.Selection); } @@ -39,3 +39,8 @@ export function reset(): void { selection = null; data = { start: 0, startOffset: 0, end: 0, endOffset: 0 }; } + +export function end(): void { + reset(); + clearTimeout(timeout); +} diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index fb876221..f8d4373e 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -11,7 +11,7 @@ let updateMap: number[] = []; let timeout: number = null; export function compute(): void { - if (timeout) { clearTimeout(timeout); } + clearTimeout(timeout); timeout = window.setTimeout(schedule, config.lookahead); } @@ -48,6 +48,7 @@ export function updates(): IBoxModel[] { updateMap = []; return summary; } + function update(id: number, box: number[]): void { let changed = true; if (id in bm) { @@ -87,3 +88,9 @@ function getLayout(x: number, y: number, element: Element): number[] { } return layout; } + +export function reset(): void { + clearTimeout(timeout); + updateMap = []; + bm = {}; +} diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 053811b5..0834f043 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -20,6 +20,7 @@ export function reset(): void { values = []; updateMap = []; changes = []; + selectorMap = []; if (DEVTOOLS_HOOK in window) { window[DEVTOOLS_HOOK] = { get, getNode, history }; } } diff --git a/src/layout/index.ts b/src/layout/index.ts index 9cde62fb..59e064d5 100644 --- a/src/layout/index.ts +++ b/src/layout/index.ts @@ -1,3 +1,4 @@ +import * as boxmodel from "@src/layout/boxmodel"; import * as discover from "@src/layout/discover"; import * as dom from "@src/layout/dom"; import * as mutation from "@src/layout/mutation"; @@ -6,9 +7,11 @@ export function start(): void { dom.reset(); mutation.start(); discover.start(); + boxmodel.reset(); } export function end(): void { dom.reset(); mutation.end(); + boxmodel.reset(); } diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index 788bc417..c74f923c 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -43,6 +43,7 @@ export function end(): void { observer = null; CSSStyleSheet.prototype.insertRule = insertRule; CSSStyleSheet.prototype.deleteRule = deleteRule; + mutations = []; } function handle(m: MutationRecord[]): void { diff --git a/src/layout/node.ts b/src/layout/node.ts index 34eb8830..e774c64f 100644 --- a/src/layout/node.ts +++ b/src/layout/node.ts @@ -2,7 +2,7 @@ import { Source } from "@clarity-types/layout"; import config from "@src/core/config"; import * as dom from "./dom"; -let ignoreAttributes = ["title", "alt", "onload", "onfocus"]; +const ignoreAttributes = ["title", "alt", "onload", "onfocus"]; export default function(node: Node, source: Source): void { // Do not track this change if we are attempting to remove a node before discovering it diff --git a/types/core.d.ts b/types/core.d.ts index 764d6e2d..e27d5201 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -3,13 +3,11 @@ import { IClarityData, IPayload, Token } from "./data"; type TaskFunction = () => Promise; type TaskResolve = () => void; -export interface IEventBindingPair { +export interface IEventBindingData { + event: string; target: EventTarget; listener: EventListener; -} - -export interface IBindingContainer { - [key: string]: IEventBindingPair[]; + capture: boolean; } export interface IConfig { diff --git a/types/data.d.ts b/types/data.d.ts index eb9a90b2..16a1c806 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -4,41 +4,41 @@ export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); export const enum Event { - Page, - Unload, - Discover, - Mutation, - BoxModel, - Checksum, - Ping, - Click, - MouseMove, - MouseDown, - MouseUp, - MouseWheel, - DoubleClick, - RightClick, - TouchStart, - TouchEnd, - TouchMove, - TouchCancel, - Selection, - Resize, - Scroll, - Change, - Document, - Visibility, - Network, - Performance, - ScriptError, - ImageError, - LayoutSummary + Page = 0, + Unload = 1, + Discover = 2, + Mutation = 3, + BoxModel = 4, + Checksum = 5, + Ping = 6, + Click = 7, + MouseMove = 8, + MouseDown = 9, + MouseUp = 10, + MouseWheel = 11, + DoubleClick = 12, + RightClick = 13, + TouchStart = 14, + TouchEnd = 15, + TouchMove = 16, + TouchCancel = 17, + Selection = 18, + Resize = 19, + Scroll = 20, + Change = 21, + Document = 22, + Visibility = 23, + Network = 24, + Performance = 25, + ScriptError = 26, + ImageError = 27, + LayoutSummary = 28 } export const enum Upload { - Async, - Beacon, - Backup + Async = 0, + Beacon = 1, + Backup = 2 } export interface IPayload { @@ -54,7 +54,8 @@ export interface ISerializedPayload { } export interface IDecodedPayload { - time: number; + timestamp: number; + ua: string; envelope: IEnvelope; metrics: IDecodedMetric; stream: IDecodedEvent[]; @@ -85,12 +86,15 @@ export interface IMetadata { } export interface IPage { + timestamp: number; + elapsed: number; url: string; title: string; referrer: string; } export interface IEnvelope { + elapsed: number; sequence: number; version: string; projectId: string; @@ -104,3 +108,8 @@ export interface IEnvelope { export interface IPing { gap: number; } + +export interface IAugmentation { + timestamp: number; + ua: string; +} diff --git a/types/metric.d.ts b/types/metric.d.ts index 771a4bea..4c19a830 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -8,30 +8,28 @@ export const enum MetricType { } export const enum Metric { - /* Counter */ - Nodes, - LayoutBytes, - InteractionBytes, - NetworkBytes, - DiagnosticBytes, - Mutations, - Interactions, - Clicks, - Selections, - Changes, - ScriptErrors, - ImageErrors, - DiscoverTime, - MutationTime, - BoxModelTime, - StartTime, - ActiveTime, - EndTime, - /* Measures */ - ViewportWidth, - ViewportHeight, - DocumentWidth, - DocumentHeight + Nodes = 0, + LayoutBytes = 1, + InteractionBytes = 2, + NetworkBytes = 3, + DiagnosticBytes = 4, + Mutations = 5, + Interactions = 6, + Clicks = 7, + Selections = 8, + Changes = 9, + ScriptErrors = 10, + ImageErrors = 11, + DiscoverTime = 12, + MutationTime = 13, + BoxModelTime = 14, + StartTime = 15, + ActiveTime = 16, + EndTime = 17, + ViewportWidth = 18, + ViewportHeight = 19, + DocumentWidth = 20, + DocumentHeight = 21 } export interface IMetric { From d1b4aa56dfd7f6b244ad6505a1606d627f9c53e1 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 23 Sep 2019 06:56:56 -0700 Subject: [PATCH 099/105] Simplifying metrics and updating version (#148) * Simplifying metrics and updating version * Adding support for event summaries * Do not send empty BoxModel event * Simplifying decoding logic and adding summaries * Fixing layout decoding bug * Simplifying metrics and support for tags * Addressing pull request feedback --- decode/clarity.ts | 56 +++++++++++++++---------- decode/data.ts | 23 +++++++++++ decode/envelope.ts | 4 +- decode/interaction.ts | 4 ++ decode/layout.ts | 51 +++++++++++++++++------ decode/metric.ts | 60 +++------------------------ decode/page.ts | 15 ------- decode/render.ts | 52 +++++++++++++++--------- decode/summary.ts | 45 +++++++++++++-------- package.json | 2 +- src/clarity.ts | 7 ++-- src/core/version.ts | 2 +- src/data/encode.ts | 10 ++++- src/data/index.ts | 3 ++ src/data/metadata.ts | 14 ++++--- src/data/ping.ts | 4 +- src/data/tag.ts | 13 ++++++ src/data/upload.ts | 8 ++-- src/interaction/encode.ts | 4 ++ src/interaction/pointer.ts | 14 ++++++- src/layout/boxmodel.ts | 20 ++++++--- src/layout/dom.ts | 10 ++--- src/layout/mutation.ts | 23 ++++++++--- src/layout/node.ts | 4 +- src/metric/encode.ts | 42 ++----------------- src/metric/index.ts | 22 +++------- types/data.d.ts | 83 +++++++++++++++++++++++--------------- types/index.d.ts | 1 + types/interaction.d.ts | 2 + types/layout.d.ts | 10 +++-- types/metric.d.ts | 49 ---------------------- 31 files changed, 338 insertions(+), 319 deletions(-) create mode 100644 decode/data.ts delete mode 100644 decode/page.ts create mode 100644 src/data/tag.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 7c78593a..42f523a7 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,25 +1,34 @@ +import version from "../src/core/version"; import { Event, IAugmentation, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; +import data from "./data"; import diagnostic from "./diagnostic"; import envelope from "./envelope"; import interaction from "./interaction"; -import layout from "./layout"; +import * as layout from "./layout"; import metric from "./metric"; -import page from "./page"; import * as r from "./render"; -import summarize from "./summary"; +import * as summary from "./summary"; let pageId: string = null; -export function decode(data: string | IPayload, augmentations: IAugmentation = null): IDecodedPayload { - let json: IPayload = typeof data === "string" ? JSON.parse(data) : data; +export function decode(input: string | IPayload, augmentations: IAugmentation = null): IDecodedPayload { + let json: IPayload = typeof input === "string" ? JSON.parse(input) : input; let timestamp = augmentations ? augmentations.timestamp : Date.now(); let ua = augmentations ? augmentations.ua : (navigator && "userAgent" in navigator ? navigator.userAgent : ""); - let payload: IDecodedPayload = { timestamp, ua, envelope: envelope(json.e), metrics: metric(json.m), stream: [], backup: [] }; + let payload: IDecodedPayload = { timestamp, ua, envelope: envelope(json.e), metrics: metric(json.m), analytics: [], playback: [] }; let encoded: Token[][] = json.d; + if (payload.envelope.version !== version) { + throw new Error(`Invalid Clarity Version. Actual: ${payload.envelope.version} | Expected: ${version}`); + } + + /* Reset components before decoding to keep them stateless */ + summary.reset(); + layout.reset(); + for (let entry of encoded) { let event: IDecodedEvent; - let summary: IDecodedEvent; + summary.decode(entry); switch (entry[1]) { case Event.Scroll: case Event.Document: @@ -38,38 +47,43 @@ export function decode(data: string | IPayload, augmentations: IAugmentation = n case Event.TouchEnd: case Event.TouchMove: event = interaction(entry); - payload.stream.push(event); + payload.analytics.push(event); break; case Event.BoxModel: - event = layout(entry); - payload.backup.push(event); + event = layout.decode(entry); + payload.playback.push(event); break; case Event.Discover: case Event.Mutation: - event = layout(entry); - summary = summarize(event); - payload.stream.push(event); - payload.stream.push(summary); + event = layout.decode(entry); + payload.playback.push(event); break; case Event.Checksum: - event = layout(entry); - payload.stream.push(event); + event = layout.decode(entry); + payload.analytics.push(event); break; case Event.Page: - event = page(entry); - payload.stream.push(event); + case Event.Ping: + case Event.Tag: + event = data(entry); + payload.analytics.push(event); break; case Event.ScriptError: case Event.ImageError: event = diagnostic(entry); - payload.stream.push(event); + payload.analytics.push(event); break; default: event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; - payload.backup.push(event); + payload.playback.push(event); break; } } + + /* Enrich decoded payload with derived events */ + payload.analytics.push(...summary.enrich()); + payload.analytics.push(...layout.enrich()); + return payload; } @@ -90,7 +104,7 @@ export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, head r.metric(decoded.metrics, header); // Replay events - let events = [...decoded.stream, ...decoded.backup].sort(sort); + let events = [...decoded.analytics, ...decoded.playback].sort(sort); replay(events, iframe); } diff --git a/decode/data.ts b/decode/data.ts new file mode 100644 index 00000000..5ffb7038 --- /dev/null +++ b/decode/data.ts @@ -0,0 +1,23 @@ +import { Event, IDecodedEvent, IPageData, IPingData, ITagData, Token } from "../types/data"; + +export default function(tokens: Token[]): IDecodedEvent { + let time = tokens[0] as number; + let event = tokens[1] as Event; + switch (event) { + case Event.Page: + let page: IPageData = { + timestamp: tokens[2] as number, + elapsed: tokens[3] as number, + url: tokens[4] as string, + title: tokens[5] as string, + referrer: tokens[6] as string + }; + return { time, event, data: page }; + case Event.Ping: + let ping: IPingData = { gap: tokens[2] as number }; + return { time, event, data: ping }; + case Event.Tag: + let tag: ITagData = { key: tokens[2] as string, value: tokens[3] as string }; + return { time, event, data: tag }; + } +} diff --git a/decode/envelope.ts b/decode/envelope.ts index ab21dc79..ac27daf8 100644 --- a/decode/envelope.ts +++ b/decode/envelope.ts @@ -1,4 +1,4 @@ -import { IEnvelope, Token, Upload } from "../types/data"; +import { Flag, IEnvelope, Token, Upload } from "../types/data"; export default function(tokens: Token[]): IEnvelope { return { @@ -10,6 +10,6 @@ export default function(tokens: Token[]): IEnvelope { sessionId: tokens[5] as string, pageId: tokens[6] as string, upload: tokens[7] as Upload, - end: tokens[8] as number + end: tokens[8] as Flag }; } diff --git a/decode/interaction.ts b/decode/interaction.ts index f162d2a3..9b4eb29b 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -17,6 +17,10 @@ export default function(tokens: Token[]): IDecodedEvent { case Event.TouchEnd: case Event.TouchMove: let pointerData: IPointer = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; + if (tokens.length > 6) { + pointerData.targetX = tokens[5] as number; + pointerData.targetY = tokens[6] as number; + } return { time, event, data: pointerData }; case Event.Resize: let resizeData: IResize = { width: tokens[2] as number, height: tokens[3] as number }; diff --git a/decode/layout.ts b/decode/layout.ts index 255aba14..1722f937 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -1,16 +1,24 @@ +import hash from "../src/data/hash"; import { resolve } from "../src/data/token"; import { Event, IDecodedEvent, Token } from "../types/data"; -import { IAttributes, IBoxModel, IChecksum, IDecodedNode, IDocumentSize, } from "../types/layout"; +import { IAttributes, IBoxModel, IChecksum, IDecodedNode, IDocumentSize, ILayout, IResource } from "../types/layout"; const ID_ATTRIBUTE = "data-clarity"; let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; -let selectorMap = {}; +let selectors: ILayout[]; +let resources: IResource[]; +let lastTime: number; -export default function(tokens: Token[]): IDecodedEvent { - let time = tokens[0] as number; +export function reset(): void { + selectors = []; + resources = []; + lastTime = null; +} + +export function decode(tokens: Token[]): IDecodedEvent { + let time = lastTime = tokens[0] as number; let event = tokens[1] as Event; let decoded: IDecodedEvent = {time, event, data: []}; - selectorMap = {}; switch (event) { case Event.Document: @@ -78,18 +86,24 @@ export default function(tokens: Token[]): IDecodedEvent { } } +export function enrich(): IDecodedEvent[] { + let output = []; + if (selectors.length > 0) { output.push({ time: lastTime, event: Event.Layout, data: selectors }); } + if (resources.length > 0) { output.push({ time: lastTime, event: Event.Resource, data: resources }); } + return output; +} + function process(node: any[] | number[], tagIndex: number): IDecodedNode { let output: IDecodedNode = { id: node[0], parent: tagIndex > 1 ? node[1] : null, next: tagIndex > 2 ? node[2] : null, - tag: node[tagIndex], - selector: "", + tag: node[tagIndex] }; let hasAttribute = false; let attributes = {}; let value = null; - let path = output.parent in selectorMap ? `${selectorMap[output.parent]}>` : null; + let path = output.parent in selectors ? `${selectors[output.parent]}>` : null; for (let i = tagIndex + 1; i < node.length; i++) { let token = node[i] as string; @@ -116,14 +130,15 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { } } - output.selector = selector(output.id, path, output.tag, attributes); + selector(output.id, output.tag, path, attributes); + resource(output.tag, attributes); if (hasAttribute) { output.attributes = attributes; } if (value) { output.value = value; } return output; } -function selector(id: number, path: string, tag: string, attributes: IAttributes): string { +function selector(id: number, tag: string, path: string, attributes: IAttributes): void { switch (tag) { case "STYLE": case "TITLE": @@ -131,14 +146,24 @@ function selector(id: number, path: string, tag: string, attributes: IAttributes case "META": case "*T": case "*D": - return ""; + break; default: let s = path && path.length > 0 ? path + tag : tag; if ("id" in attributes) { s = `${tag}#${attributes["id"]}`; } if ("class" in attributes) { s += `.${attributes["class"].trim().split(" ").join(".")}`; } if (ID_ATTRIBUTE in attributes) { s = `*${attributes[ID_ATTRIBUTE]}`; } - selectorMap[id] = s; - return s; + if (s) { selectors.push({ id, checksum: hash(s), selector: s }); } + break; + } +} + +function resource(tag: string, attributes: IAttributes): void { + switch (tag) { + case "LINK": + if ("href" in attributes && "rel" in attributes && attributes["rel"] === "stylesheet") { + resources.push({ tag, url: attributes["href"]}); + } + break; } } diff --git a/decode/metric.ts b/decode/metric.ts index 351621b8..16ae94ba 100644 --- a/decode/metric.ts +++ b/decode/metric.ts @@ -1,61 +1,13 @@ -import { Event, Token } from "../types/data"; -import { IDecodedMetric, IMetricMap, Metric, MetricType } from "../types/metric"; +import { Token } from "../types/data"; +import { IMetric } from "../types/metric"; -let metricMap: IMetricMap = {}; +let metrics: IMetric = null; -metricMap[Metric.Nodes] = { name: "Node Count", unit: ""}; -metricMap[Metric.LayoutBytes] = { name: "Layout Bytes", unit: "KB"}; -metricMap[Metric.InteractionBytes] = { name: "Interaction Bytes", unit: "KB"}; -metricMap[Metric.NetworkBytes] = { name: "Network Bytes", unit: "KB"}; -metricMap[Metric.DiagnosticBytes] = { name: "Diagnostic Bytes", unit: "KB"}; -metricMap[Metric.Mutations] = { name: "Mutation Count", unit: ""}; -metricMap[Metric.Interactions] = { name: "Interaction Count", unit: ""}; -metricMap[Metric.Clicks] = { name: "Click Count", unit: ""}; -metricMap[Metric.Selections] = { name: "Selection Count", unit: ""}; -metricMap[Metric.ScriptErrors] = { name: "Script Errors", unit: ""}; -metricMap[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; -metricMap[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; -metricMap[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; -metricMap[Metric.BoxModelTime] = { name: "Box Model Time", unit: "ms"}; -metricMap[Metric.StartTime] = { name: "Start Time", unit: "s"}; -metricMap[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; -metricMap[Metric.EndTime] = { name: "End Time", unit: "s"}; -metricMap[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px"}; -metricMap[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; -metricMap[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; -metricMap[Metric.DocumentHeight] = { name: "Document Height", unit: "px"}; - -let metrics: IDecodedMetric = null; - -export default function(tokens: Token[]): IDecodedMetric { +export default function(tokens: Token[]): IMetric { let i = 0; - let metricType = null; - metrics = { counters: {}, measures: {}, events: [], marks: [] }; + metrics = {}; while (i < tokens.length) { - // Determine metric time for subsequent processing - if (typeof(tokens[i]) === "string") { - metricType = tokens[i++]; - continue; - } - - // Parse metrics - switch (metricType) { - case MetricType.Counter: - let counter = metricMap[tokens[i++] as Metric]; - metrics.counters[counter.name] = { value: tokens[i++] as number, unit: counter.unit }; - break; - case MetricType.Measure: - let measure = metricMap[tokens[i++] as Metric]; - metrics.measures[measure.name] = { value: tokens[i++] as number, unit: measure.unit }; - break; - case MetricType.Event: - metrics.events.push({ event: tokens[i++] as Event, time: tokens[i++] as number, duration: tokens[i++] as number }); - break; - case MetricType.Marks: - metrics.marks.push({ key: tokens[i++] as string, value: tokens[i++] as string, time: tokens[i++] as number }); - break; - } + metrics[tokens[i++] as number] = tokens[i++] as number; } - return metrics; } diff --git a/decode/page.ts b/decode/page.ts deleted file mode 100644 index 83d34e0e..00000000 --- a/decode/page.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Event, IDecodedEvent, Token } from "../types/data"; - -export default function(tokens: Token[]): IDecodedEvent { - return { - time: tokens[0] as number, - event: tokens[1] as Event, - data: { - timestamp: tokens[2] as number, - elapsed: tokens[3] as number, - url: tokens[4] as string, - title: tokens[5] as string, - referrer: tokens[6] as string - } - }; -} diff --git a/decode/render.ts b/decode/render.ts index 4d4e7cd4..69016858 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,13 +1,35 @@ import { Event } from "../types/data"; import { IChange, IPointer, IResize, IScroll, ISelection } from "../types/interaction"; import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; -import { IDecodedMetric } from "../types/metric"; +import { IMetric, Metric } from "../types/metric"; let nodes = {}; let boxmodels = {}; -let metrics: IDecodedMetric = null; +let metrics: IMetric = null; let svgns: string = "http://www.w3.org/2000/svg"; let lean = false; +const METRIC_MAP = {}; +METRIC_MAP[Metric.Nodes] = { name: "Node Count", unit: ""}; +METRIC_MAP[Metric.LayoutBytes] = { name: "Layout Bytes", unit: "KB"}; +METRIC_MAP[Metric.InteractionBytes] = { name: "Interaction Bytes", unit: "KB"}; +METRIC_MAP[Metric.NetworkBytes] = { name: "Network Bytes", unit: "KB"}; +METRIC_MAP[Metric.DiagnosticBytes] = { name: "Diagnostic Bytes", unit: "KB"}; +METRIC_MAP[Metric.Mutations] = { name: "Mutation Count", unit: ""}; +METRIC_MAP[Metric.Interactions] = { name: "Interaction Count", unit: ""}; +METRIC_MAP[Metric.Clicks] = { name: "Click Count", unit: ""}; +METRIC_MAP[Metric.Selections] = { name: "Selection Count", unit: ""}; +METRIC_MAP[Metric.ScriptErrors] = { name: "Script Errors", unit: ""}; +METRIC_MAP[Metric.ImageErrors] = { name: "Image Errors", unit: ""}; +METRIC_MAP[Metric.DiscoverTime] = { name: "Discover Time", unit: "ms"}; +METRIC_MAP[Metric.MutationTime] = { name: "Mutation Time", unit: "ms"}; +METRIC_MAP[Metric.BoxModelTime] = { name: "Box Model Time", unit: "ms"}; +METRIC_MAP[Metric.StartTime] = { name: "Start Time", unit: "s"}; +METRIC_MAP[Metric.ActiveTime] = { name: "Active Time", unit: "ms"}; +METRIC_MAP[Metric.EndTime] = { name: "End Time", unit: "s"}; +METRIC_MAP[Metric.ViewportWidth] = { name: "Viewport Width", unit: "px"}; +METRIC_MAP[Metric.ViewportHeight] = { name: "Viewport Height", unit: "px"}; +METRIC_MAP[Metric.DocumentWidth] = { name: "Document Width", unit: "px"}; +METRIC_MAP[Metric.DocumentHeight] = { name: "Document Height", unit: "px"}; // tslint:disable-next-line: max-line-length let pointerIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S912AAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6JQAAgIMAAPn/AACA6AAAdTAAAOpgAAA6lwAAF2+XqZnUAAACaUlEQVR4nGL8f58BHYgAsT8Q2wGxBBAzQcX/AfFrID4CxOuA+BWKLoX/YAoggBjRDHQD4ngglgRiPgyrIOAzEL8E4lVQg1EMBAggFiSFYUAcA8RSOAyCAV4oTgViTiBeiiwJEEAw71gRaRgyEAXiKCB2RBYECCCQgcIMEG+SYhgMiANxEhDzwwQAAghkoAMQK5NhGAwoALE1jAMQQCADQU7mpMBAZqijwAAggEAGqgAxOwUGskHNAAOAAAIZyEtIh4INg3bfHHD6xAUEYAyAAAIZ+IuQgU9fMLCXdzDIzV3JIIhDyQ8YAyCAQAaCUv8/fAZysDP8+/OXgTG7jkFhwRoMQ0F6n8M4AAEEMvAKsg34wM9fDEwgQ1dtRSQTIPgNxFdhHIAAAhm4AYg/EmMgCHz7zsCUVMaguHob3FCQYzbD5AECCGTgJSDeCbWJKPD1GwNzSjmD4tZ9DFxgvQr/b8PkAAIIlvVWA/FuUgz99IWBOTyXQcE+nOEOsjhAACGXNnJAHAnE9kAshqyIV5vB4Ms3cALGBkAlj9////9PgTgAAcSEJPEIiDuBeBYQP2CAhOt3BsLJCpSfNzAyMpqDOAABhF4ewh3FAMmf2kAsyqnBUPDjJ8HcdBvoSjWAAGIEEgTUMTAAbf/AwICSVGCgD4hPgJQA8WegWdsBAogFiyJC4C0QgxI3KLj4gIasRpYECCAGkAsJYSAAuRDEAKUEQwZIzgDxvwCxCrJagAAi1kAQAYpFESh/BlQMhJuR1QIEELEGlgOxHBLflAGSh0Gc60DMBpMDCCCiDMRhyXoGSJUaDgpPmDhAgAEAN5Ugk0bMYNIAAAAASUVORK5CYII="; @@ -19,27 +41,21 @@ let touchIcon = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAYCAYAAAD6S export function reset(): void { nodes = {}; boxmodels = {}; - metrics = { counters: {}, measures: {}, events: [], marks: [] }; + metrics = {}; } -export function metric(data: IDecodedMetric, header: HTMLElement): void { +export function metric(data: IMetric, header: HTMLElement): void { let html = []; - // Copy over counters - for (let counter in data.counters) { - if (data.counters[counter]) { metrics.counters[counter] = data.counters[counter]; } + // Copy over metrics for future reference + for (let m in data) { + if (data[m]) { metrics[m] = data[m]; } } - - // Copy over measures - for (let measure in data.measures) { - if (data.measures[measure]) { metrics.measures[measure] = data.measures[measure]; } - } - - let entries = {...metrics.counters, ...metrics.measures}; - for (let entry in entries) { - if (entries[entry]) { - let m = entries[entry]; - html.push(`

  • ${value(m.value, m.unit)}${m.unit}

    ${entry}
    • `); + for (let entry in metrics) { + if (metrics[entry]) { + let m = metrics[entry]; + let map = METRIC_MAP[entry]; + html.push(`

  • ${value(m, map.unit)}${map.unit}

    ${map.name}
    • `); } } diff --git a/decode/summary.ts b/decode/summary.ts index 50442510..5377f27f 100644 --- a/decode/summary.ts +++ b/decode/summary.ts @@ -1,21 +1,32 @@ -import hash from "../src/data/hash"; -import { Event, IDecodedEvent } from "../types/data"; -import { IDecodedNode, ILayoutSummary } from "../types/layout"; +import { Event, IDecodedEvent, IEventSummary, Token } from "../types/data"; -export default function(event: IDecodedEvent): IDecodedEvent { - switch (event.event) { - case Event.Discover: - case Event.Mutation: - let checksumMap: IDecodedEvent = {time: event.time, event: Event.LayoutSummary, data: []}; - let nodes: IDecodedNode[] = event.data; - for (let node of nodes) { - // Do not track nodes where we don't have a valid selector - e.g. text nodes - if (node.selector && node.selector.length > 0) { - let checksum = hash(node.selector); - let data: ILayoutSummary = { id: node.id, checksum, selector: node.selector }; - checksumMap.data.push(data); - } +let summary: { [key: number]: IEventSummary[] } = null; +const SUMMARY_THRESHOLD = 250; + +export function reset(): void { + summary = {}; +} + +export function decode(entry: Token[]): void { + let time = entry[0] as number; + let type = entry[1] as Event; + let data: IEventSummary = { event: type, start: time, end: time }; + if (!(type in summary)) { summary[type] = [data]; } + + let s = summary[type][summary[type].length - 1]; + if (time - s.end < SUMMARY_THRESHOLD) { s.end = time; } else { summary[type].push(data); } +} + +export function enrich(): IDecodedEvent[] { + let data: IEventSummary[] = []; + let time = null; + for (let type in summary) { + if (summary[type]) { + for (let d of summary[type]) { + time = time ? Math.min(time, d.start) : d.start; + data.push(d); } - return checksumMap; + } } + return data.length > 0 ? [{ time, event: Event.Summary, data }] : []; } diff --git a/package.json b/package.json index d3b5ab58..770123fc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.4", + "version": "1.0.0-beta.5", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/clarity.ts b/src/clarity.ts index 07384102..3eb0a4e7 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -5,8 +5,9 @@ import { bind } from "@src/core/event"; import * as data from "@src/data"; import * as diagnostic from "@src/diagnostic"; import * as interaction from "@src/interaction"; -import * as dom from "@src/layout"; +import * as layout from "@src/layout"; import * as metric from "@src/metric"; +export { tag } from "@src/data/tag"; let status = false; @@ -28,7 +29,7 @@ export function start(override: IConfig = {}): void { metric.start(); data.start(); diagnostic.start(); - dom.start(); + layout.start(); interaction.start(); } } @@ -49,7 +50,7 @@ export function resume(): void { export function end(): void { if (status) { interaction.end(); - dom.end(); + layout.end(); diagnostic.end(); data.end(); metric.end(); diff --git a/src/core/version.ts b/src/core/version.ts index 24e7307b..d6fbc747 100644 --- a/src/core/version.ts +++ b/src/core/version.ts @@ -1,2 +1,2 @@ -let version = "1.0.0"; +let version = "1.0.0-b5"; export default version; diff --git a/src/data/encode.ts b/src/data/encode.ts index 00a8cc28..63a5a42c 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -2,7 +2,8 @@ import {Event, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import time from "@src/core/time"; import { metadata } from "@src/data/metadata"; -import { data as pingdata } from "@src/data/ping"; +import * as ping from "@src/data/ping"; +import * as tag from "@src/data/tag"; import * as metric from "@src/metric"; import { queue } from "./upload"; @@ -11,7 +12,7 @@ export default function(event: Event): void { let tokens: Token[] = [t, event]; switch (event) { case Event.Ping: - tokens.push(pingdata.gap); + tokens.push(ping.data.gap); queue(tokens); break; case Event.Page: @@ -23,5 +24,10 @@ export default function(event: Event): void { tokens.push(metadata.page.referrer); queue(tokens); break; + case Event.Tag: + tokens.push(tag.data.key); + tokens.push(tag.data.value); + queue(tokens); + break; } } diff --git a/src/data/index.ts b/src/data/index.ts index 7a6b8ba0..296fa189 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -1,14 +1,17 @@ import * as metadata from "@src/data/metadata"; import * as ping from "@src/data/ping"; +import * as tag from "@src/data/tag"; import * as upload from "@src/data/upload"; export function start(): void { upload.start(); metadata.start(); ping.start(); + tag.reset(); } export function end(): void { + tag.reset(); ping.end(); upload.end(); metadata.end(); diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 5294ec56..cce09572 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,4 +1,4 @@ -import { Event, ICookieData, IEnvelope, IMetadata, IPage, Token, Upload } from "@clarity-types/data"; +import { Event, Flag, ICookieData, IEnvelope, IMetadata, IPageData, Token, Upload } from "@clarity-types/data"; import config from "@src/core/config"; import time from "@src/core/time"; import version from "@src/core/version"; @@ -13,12 +13,13 @@ export let metadata: IMetadata = null; export function start(): void { let cookie: ICookieData = read(); let ts = Date.now(); + let elapsed = time(); let projectId = config.projectId || hash(location.host); let userId = cookie && cookie.userId ? cookie.userId : guid(); let sessionId = cookie && cookie.sessionId && ts - cookie.timestamp < CLARITY_SESSION_LENGTH ? cookie.sessionId : ts.toString(36); let pageId = guid(); - let e: IEnvelope = { elapsed: time(), sequence: 0, version, pageId, userId, sessionId, projectId, upload: Upload.Async, end: 0 }; - let p: IPage = { timestamp: ts, elapsed: time(), url: location.href, title: document.title, referrer: document.referrer }; + let e: IEnvelope = { elapsed, sequence: 0, version, pageId, userId, sessionId, projectId, upload: Upload.Async, end: Flag.False }; + let p: IPageData = { timestamp: ts, elapsed, url: location.href, title: document.title, referrer: document.referrer }; metadata = { page: p, envelope: e }; @@ -32,14 +33,15 @@ export function end(): void { } export function envelope(last: boolean, backup: boolean = false): Token[] { - let upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); let e = metadata.envelope; - if (upload !== Upload.Backup) { + e.upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); + e.end = last ? Flag.True : Flag.False; + if (e.upload !== Upload.Backup) { e.elapsed = time(); e.sequence++; } - return [e.elapsed, e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, upload, last ? 1 : 0]; + return [e.elapsed, e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, e.upload, e.end]; } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript diff --git a/src/data/ping.ts b/src/data/ping.ts index 97054144..f57f5695 100644 --- a/src/data/ping.ts +++ b/src/data/ping.ts @@ -1,10 +1,10 @@ -import { Event, IPing } from "@clarity-types/data"; +import { Event, IPingData } from "@clarity-types/data"; import { pause } from "@src/clarity"; import config from "@src/core/config"; import time from "@src/core/time"; import encode from "./encode"; -export let data: IPing; +export let data: IPingData; let last = 0; let interval = 0; let timeout: number = null; diff --git a/src/data/tag.ts b/src/data/tag.ts new file mode 100644 index 00000000..9d2d5e6f --- /dev/null +++ b/src/data/tag.ts @@ -0,0 +1,13 @@ +import { Event, ITagData } from "@clarity-types/data"; +import encode from "@src/data/encode"; + +export let data: ITagData = null; + +export function reset(): void { + data = null; +} + +export function tag(key: string, value: string): void { + data = { key, value }; + encode(Event.Tag); +} diff --git a/src/data/upload.ts b/src/data/upload.ts index b8956fad..ea14bd61 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,4 +1,4 @@ -import { Event, ISerializedPayload, Token } from "@clarity-types/data"; +import { Event, IEncodedPayload, Token } from "@clarity-types/data"; import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import {envelope} from "@src/data/metadata"; @@ -52,14 +52,14 @@ export function end(): void { function upload(last: boolean = false): void { let handler = config.upload ? config.upload : send; - let payload: ISerializedPayload = {e: JSON.stringify(envelope(last)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; + let payload: IEncodedPayload = {e: JSON.stringify(envelope(last)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; handler(stringify(payload), last); if (last) { backup(payload); } events = []; if (!last) { ping.reset(); } } -function stringify(payload: ISerializedPayload): string { +function stringify(payload: IEncodedPayload): string { return `{"e":${payload.e},"m":${payload.m},"d":${payload.d}}`; } @@ -84,7 +84,7 @@ function recover(): void { } } -function backup(payload: ISerializedPayload): void { +function backup(payload: IEncodedPayload): void { if (typeof localStorage !== "undefined") { payload.e = JSON.stringify(envelope(true, true)); localStorage.setItem("clarity-backup", stringify(payload)); diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 3ed302c2..91180a79 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -32,6 +32,10 @@ export default function(type: Event): void { tokens.push(entry.target); tokens.push(entry.x); tokens.push(entry.y); + if (entry.targetX && entry.targetY) { + tokens.push(entry.targetX); + tokens.push(entry.targetY); + } queue(tokens); } pointer.reset(); diff --git a/src/interaction/pointer.ts b/src/interaction/pointer.ts index b089bd76..79df4986 100644 --- a/src/interaction/pointer.ts +++ b/src/interaction/pointer.ts @@ -3,6 +3,7 @@ import { IPointer } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; +import * as boxmodel from "@src/layout/boxmodel"; import { getId } from "@src/layout/dom"; import encode from "./encode"; @@ -28,8 +29,17 @@ function mouse(event: Event, evt: MouseEvent): void { let x = "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null); let y = "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null); let target = evt.target ? getId(evt.target as Node) : null; - event = event === Event.Click && (evt.buttons === 2 || evt.button === 2) ? Event.RightClick : event; - handler(event, {target, x, y, time: time()}); + let targetX = null; // x coordinate relative to the target element + let targetY = null; // y coordinate relative to the target element + if (event === Event.Click) { + // Populate (x,y) relative to target only when the above condition is met + // It's an expensive operation and we can expand the scope as desired later + event = evt.buttons === 2 || evt.button === 2 ? Event.RightClick : event; + let relative = boxmodel.relative(x, y, evt.target as Element); + targetX = relative[0]; + targetY = relative[1]; + } + handler(event, {target, x, y, targetX, targetY, time: time()}); } function touch(event: Event, evt: TouchEvent): void { diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index f8d4373e..ef9f3b8b 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -33,10 +33,10 @@ async function boxmodel(): Promise { x = "pageXOffset" in window ? window.pageXOffset : doc.scrollLeft; y = "pageYOffset" in window ? window.pageYOffset : doc.scrollTop; } - update(value.id, getLayout(x, y, dom.getNode(value.id) as Element)); + update(value.id, layout(dom.getNode(value.id) as Element, x, y)); } - await encode(Event.BoxModel); + if (updateMap.length > 0) { await encode(Event.BoxModel); } task.stop(timer); } @@ -49,6 +49,14 @@ export function updates(): IBoxModel[] { return summary; } +export function relative(x: number, y: number, element: Element): number[] { + if (x && x >= 0 && y && y >= 0 && element) { + let box = layout(element); + return [x - box[0], y - box[1]]; + } + return [null, null]; +} + function update(id: number, box: number[]): void { let changed = true; if (id in bm) { @@ -69,8 +77,8 @@ function update(id: number, box: number[]): void { } } -function getLayout(x: number, y: number, element: Element): number[] { - let layout: number[] = [0, 0, 0, 0]; +function layout(element: Element, x: number = 0, y: number = 0): number[] { + let box: number[] = [0, 0, 0, 0]; let rect = element.getBoundingClientRect(); if (rect && rect.width > 0 && rect.height > 0) { @@ -79,14 +87,14 @@ function getLayout(x: number, y: number, element: Element): number[] { // Also: using Math.floor() instead of Math.round() below because in Edge, // getBoundingClientRect returns partial pixel values (e.g. 162.5px) and Chrome already // floors the value (e.g. 162px). Keeping behavior consistent across - layout = [ + box = [ Math.floor(rect.left + x), Math.floor(rect.top + y), Math.floor(rect.width), Math.floor(rect.height) ]; } - return layout; + return box; } export function reset(): void { diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 0834f043..c6e83bc2 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -57,7 +57,7 @@ export function add(node: Node, data: INodeData, source: Source): void { children: [], data, selector: selector(id, data, parent ? parent.selector : ""), - metadata: { active: true, layout: false, masked } + metadata: { active: true, boxmodel: false, masked } }; layout(data.tag, id, parentId); track(id, source); @@ -144,7 +144,7 @@ export function has(node: Node): boolean { export function boxmodel(): INodeValue[] { let v = []; for (let id in values) { - if (values[id].metadata.active && values[id].metadata.layout) { + if (values[id].metadata.active && values[id].metadata.boxmodel) { v.push(values[id]); } } @@ -218,7 +218,7 @@ function layout(tag: string, id: number, parentId: number): void { for (let i = 0; i < value.length; i++) { let code = value.charCodeAt(i); if (!(code === 32 || code === 10 || code === 9 || code === 13)) { - values[parentId].metadata.layout = true; + values[parentId].metadata.boxmodel = true; break; } } @@ -227,11 +227,11 @@ function layout(tag: string, id: number, parentId: number): void { case "IMG": case "IFRAME": case "svg:svg": - values[id].metadata.layout = true; + values[id].metadata.boxmodel = true; break; default: // Capture layout for any element with a user defined selector - values[id].metadata.layout = values[id].selector.indexOf("*") === 0; + values[id].metadata.boxmodel = values[id].selector.indexOf("*") === 0; break; } } diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index c74f923c..674268ac 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -10,8 +10,8 @@ import processNode from "./node"; let observer: MutationObserver; let mutations: MutationRecord[] = []; -let insertRule: (rule: string, index?: number) => number; -let deleteRule: (index?: number) => void; +let insertRule: (rule: string, index?: number) => number = null; +let deleteRule: (index?: number) => void = null; export function start(): void { if (observer) { @@ -19,8 +19,8 @@ export function start(): void { } observer = window["MutationObserver"] ? new MutationObserver(handle) : null; observer.observe(document, { attributes: true, childList: true, characterData: true, subtree: true }); - insertRule = CSSStyleSheet.prototype.insertRule; - deleteRule = CSSStyleSheet.prototype.deleteRule; + if (insertRule === null) { insertRule = CSSStyleSheet.prototype.insertRule; } + if (deleteRule === null) { deleteRule = CSSStyleSheet.prototype.deleteRule; } // Some popular open source libraries, like styled-components, optimize performance // by injecting CSS using insertRule API vs. appending text node. A side effect of @@ -41,8 +41,19 @@ export function start(): void { export function end(): void { observer.disconnect(); observer = null; - CSSStyleSheet.prototype.insertRule = insertRule; - CSSStyleSheet.prototype.deleteRule = deleteRule; + + // Restoring original insertRule + if (insertRule !== null) { + CSSStyleSheet.prototype.insertRule = insertRule; + insertRule = null; + } + + // Restoring original deleteRule + if (deleteRule !== null) { + CSSStyleSheet.prototype.deleteRule = deleteRule; + deleteRule = null; + } + mutations = []; } diff --git a/src/layout/node.ts b/src/layout/node.ts index e774c64f..d7bc9ca2 100644 --- a/src/layout/node.ts +++ b/src/layout/node.ts @@ -2,7 +2,7 @@ import { Source } from "@clarity-types/layout"; import config from "@src/core/config"; import * as dom from "./dom"; -const ignoreAttributes = ["title", "alt", "onload", "onfocus"]; +const IGNORE_ATTRIBUTES = ["title", "alt", "onload", "onfocus"]; export default function(node: Node, source: Source): void { // Do not track this change if we are attempting to remove a node before discovering it @@ -96,7 +96,7 @@ function getAttributes(attributes: NamedNodeMap): {[key: string]: string} { if (attributes && attributes.length > 0) { for (let i = 0; i < attributes.length; i++) { let name = attributes[i].name; - if (ignoreAttributes.indexOf(name) < 0) { + if (IGNORE_ATTRIBUTES.indexOf(name) < 0) { output[name] = attributes[i].value; } } diff --git a/src/metric/encode.ts b/src/metric/encode.ts index bddec750..822ddbe3 100644 --- a/src/metric/encode.ts +++ b/src/metric/encode.ts @@ -1,54 +1,20 @@ import {Token} from "@clarity-types/data"; -import { MetricType } from "@clarity-types/metric"; import { metrics, reset, updates } from "@src/metric"; export default function(last: boolean = false): Token[] { let output = []; - // Encode counter metrics - output.push(MetricType.Counter); - let counters = metrics.counters; - for (let metric in counters) { - if (counters[metric]) { + // Encode metrics + for (let metric in metrics) { + if (metrics[metric]) { let m = num(metric); if (updates.indexOf(m) >= 0 || last) { output.push(m); - output.push(counters[metric]); + output.push(metrics[metric]); } } } - // Encode summary metrics - output.push(MetricType.Measure); - let measures = metrics.measures; - for (let metric in measures) { - if (measures[metric]) { - let m = num(metric); - if (updates.indexOf(m) >= 0 || last) { - output.push(m); - output.push(measures[metric]); - } - } - } - - // Encode events summary - if (metrics.events.length > 0) { output.push(MetricType.Event); } - let events = metrics.events; - for (let event of events) { - output.push(event.event); - output.push(event.time); - output.push(event.duration); - } - - // Encode user specified marks - if (metrics.marks.length > 0) { output.push(MetricType.Marks); } - let marks = metrics.marks; - for (let mark of marks) { - output.push(mark.key); - output.push(mark.value); - output.push(mark.time); - } - reset(); return output; diff --git a/src/metric/index.ts b/src/metric/index.ts index 2e7a9d94..f2c521bc 100644 --- a/src/metric/index.ts +++ b/src/metric/index.ts @@ -1,12 +1,10 @@ -import { Event } from "@clarity-types/data"; import { IMetric, Metric } from "@clarity-types/metric"; -import time from "@src/core/time"; export let metrics: IMetric = null; export let updates: Metric[] = []; export function start(): void { - metrics = { counters: {}, measures: {}, events: [], marks: [] }; + metrics = {}; } export function end(): void { @@ -14,25 +12,17 @@ export function end(): void { } export function counter(metric: Metric, increment: number = 1): void { - if (!(metric in metrics.counters)) { metrics.counters[metric] = 0; } - metrics.counters[metric] += increment; + if (!(metric in metrics)) { metrics[metric] = 0; } + metrics[metric] += increment; track(metric); } export function measure(metric: Metric, value: number): void { - if (!(metric in metrics.measures)) { metrics.measures[metric] = 0; } - metrics.measures[metric] = Math.max(value, metrics.measures[metric]); + if (!(metric in metrics)) { metrics[metric] = 0; } + metrics[metric] = Math.max(value, metrics[metric]); track(metric); } -export function event(evt: Event, begin: number, duration: number = 0): void { - metrics.events.push({ event: evt, time: begin, duration }); -} - -export function mark(key: string, value: string): void { - metrics.marks.push({ key, value, time: time() }); -} - function track(metric: Metric): void { if (updates.indexOf(metric) === -1) { updates.push(metric); @@ -41,6 +31,4 @@ function track(metric: Metric): void { export function reset(): void { updates = []; - metrics.events = []; - metrics.marks = []; } diff --git a/types/data.d.ts b/types/data.d.ts index 16a1c806..a16a6ae1 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,4 +1,4 @@ -import { IDecodedMetric } from "./metric"; +import { IMetric } from "./metric"; export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); @@ -10,29 +10,32 @@ export const enum Event { Mutation = 3, BoxModel = 4, Checksum = 5, - Ping = 6, - Click = 7, - MouseMove = 8, - MouseDown = 9, - MouseUp = 10, - MouseWheel = 11, - DoubleClick = 12, - RightClick = 13, - TouchStart = 14, - TouchEnd = 15, - TouchMove = 16, - TouchCancel = 17, - Selection = 18, - Resize = 19, - Scroll = 20, - Change = 21, - Document = 22, - Visibility = 23, - Network = 24, - Performance = 25, - ScriptError = 26, - ImageError = 27, - LayoutSummary = 28 + Tag = 6, + Ping = 7, + Click = 8, + MouseMove = 9, + MouseDown = 10, + MouseUp = 11, + MouseWheel = 12, + DoubleClick = 13, + RightClick = 14, + TouchStart = 15, + TouchEnd = 16, + TouchMove = 17, + TouchCancel = 18, + Selection = 19, + Resize = 20, + Scroll = 21, + Change = 22, + Document = 23, + Visibility = 24, + Network = 25, + Performance = 26, + ScriptError = 27, + ImageError = 28, + Layout = 29, + Resource = 30, + Summary = 31 } export const enum Upload { @@ -41,13 +44,18 @@ export const enum Upload { Backup = 2 } +export const enum Flag { + False = 0, + True = 1 +} + export interface IPayload { e: Token[]; m: Token[]; d: Token[][]; } -export interface ISerializedPayload { +export interface IEncodedPayload { e: string; m: string; d: string; @@ -57,9 +65,9 @@ export interface IDecodedPayload { timestamp: number; ua: string; envelope: IEnvelope; - metrics: IDecodedMetric; - stream: IDecodedEvent[]; - backup: IDecodedEvent[]; + metrics: IMetric; + analytics: IDecodedEvent[]; + playback: IDecodedEvent[]; } export interface IDecodedEvent { @@ -81,11 +89,11 @@ export interface IClarityData { } export interface IMetadata { - page: IPage; + page: IPageData; envelope: IEnvelope; } -export interface IPage { +export interface IPageData { timestamp: number; elapsed: number; url: string; @@ -102,14 +110,25 @@ export interface IEnvelope { sessionId: string; pageId: string; upload: Upload; - end: number; + end: Flag; } -export interface IPing { +export interface IPingData { gap: number; } +export interface ITagData { + key: string; + value: string; +} + export interface IAugmentation { timestamp: number; ua: string; } + +export interface IEventSummary { + event: Event; + start: number; + end: number; +} diff --git a/types/index.d.ts b/types/index.d.ts index ef6124e1..5e9b5fd4 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -8,6 +8,7 @@ interface IClarityJs { resume: () => void; end: () => void; active: () => boolean; + tag: (key: string, value: string) => void; } declare const clarity: IClarityJs; diff --git a/types/interaction.d.ts b/types/interaction.d.ts index fecae144..1fe13a57 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -2,6 +2,8 @@ export interface IPointer { target: number; x: number; y: number; + targetX?: number; + targetY?: number; time?: number; } diff --git a/types/layout.d.ts b/types/layout.d.ts index 3f5cd279..88c259bb 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -29,7 +29,7 @@ export interface INodeValue { export interface INodeMetadata { active: boolean; - layout: boolean; + boxmodel: boolean; masked: boolean; } @@ -44,7 +44,6 @@ export interface IDecodedNode { parent: number; next: number; tag: string; - selector: string; attributes?: IAttributes; value?: string; } @@ -64,8 +63,13 @@ export interface IChecksum { checksum: string; } -export interface ILayoutSummary { +export interface ILayout { id: number; checksum: string; selector: string; } + +export interface IResource { + tag: string; + url: string; +} diff --git a/types/metric.d.ts b/types/metric.d.ts index 4c19a830..316ada16 100644 --- a/types/metric.d.ts +++ b/types/metric.d.ts @@ -1,12 +1,5 @@ import { Event } from "./data"; -export const enum MetricType { - Counter = "C", - Measure = "M", - Event = "E", - Marks = "K" -} - export const enum Metric { Nodes = 0, LayoutBytes = 1, @@ -33,47 +26,5 @@ export const enum Metric { } export interface IMetric { - counters: IMetricValue; - measures: IMetricValue; - events: IEventMetric[]; - marks: IMarkMetric[]; -} - -export interface IMetricValue { [key: number]: number; } - -export interface IEventMetric { - event: Event; - time: number; - duration: number; -} - -export interface IMarkMetric { - key: string; - value: string; - time: number; -} - -export interface IMetricMap { - [key: number]: IMetricMapValue; -} - -export interface IMetricMapValue { - name: string; - unit: string; -} - -export interface IDecodedMetric { - counters: IDecodedMetricValue; - measures: IDecodedMetricValue; - events: IEventMetric[]; - marks: IMarkMetric[]; -} - -export interface IDecodedMetricValue { - [key: string]: { - value: number; - unit: string; - }; -} From 455649504842112fdd3dbded1d2bbbdf820cef56 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Mon, 30 Sep 2019 06:47:49 -0700 Subject: [PATCH 100/105] vNextBeta6: Type refactoring and addressing known issues (#149) * Strongly typed decoded payload * Drop "I" from interfaces per Typescript guidance * Add retry logic to XHR uploads * Addressing PR feedback * Rename visible to visibility --- decode/clarity.ts | 179 ++++++++++++++++-------- decode/data.ts | 72 ++++++++-- decode/diagnostic.ts | 11 +- decode/envelope.ts | 15 -- decode/interaction.ts | 27 ++-- decode/layout.ts | 72 +++++----- decode/metric.ts | 13 -- decode/render.ts | 32 ++--- decode/summary.ts | 32 ----- package.json | 2 +- src/clarity.ts | 9 +- src/core/config.ts | 4 +- src/core/event.ts | 4 +- src/core/task.ts | 12 +- src/core/version.ts | 2 +- src/data/encode.ts | 32 ++++- src/data/index.ts | 3 + src/data/metadata.ts | 28 ++-- src/{metric/index.ts => data/metric.ts} | 21 +-- src/data/ping.ts | 4 +- src/data/tag.ts | 4 +- src/data/upload.ts | 61 +++++--- src/diagnostic/encode.ts | 5 +- src/diagnostic/image.ts | 4 +- src/diagnostic/script.ts | 4 +- src/interaction/change.ts | 6 +- src/interaction/encode.ts | 7 +- src/interaction/pointer.ts | 10 +- src/interaction/resize.ts | 4 +- src/interaction/scroll.ts | 12 +- src/interaction/selection.ts | 4 +- src/interaction/unload.ts | 4 +- src/interaction/visibility.ts | 4 +- src/layout/boxmodel.ts | 13 +- src/layout/discover.ts | 5 +- src/layout/document.ts | 21 +-- src/layout/dom.ts | 26 ++-- src/layout/encode.ts | 21 ++- src/layout/index.ts | 3 + src/layout/mutation.ts | 5 +- src/metric/encode.ts | 25 ---- tslint.json | 1 + types/core.d.ts | 24 ++-- types/data.d.ts | 150 +++++++++++--------- types/decode/core.d.ts | 6 + types/decode/data.d.ts | 13 ++ types/decode/decode.d.ts | 34 +++++ types/decode/diagnostic.d.ts | 8 ++ types/decode/index.d.ts | 16 +++ types/decode/interaction.d.ts | 13 ++ types/decode/layout.d.ts | 21 +++ types/diagnostic.d.ts | 6 +- types/index.d.ts | 12 +- types/interaction.d.ts | 27 ++-- types/layout.d.ts | 48 +++---- types/metric.d.ts | 30 ---- 56 files changed, 709 insertions(+), 522 deletions(-) delete mode 100644 decode/envelope.ts delete mode 100644 decode/metric.ts delete mode 100644 decode/summary.ts rename src/{metric/index.ts => data/metric.ts} (52%) delete mode 100644 src/metric/encode.ts create mode 100644 types/decode/core.d.ts create mode 100644 types/decode/data.d.ts create mode 100644 types/decode/decode.d.ts create mode 100644 types/decode/diagnostic.d.ts create mode 100644 types/decode/index.d.ts create mode 100644 types/decode/interaction.d.ts create mode 100644 types/decode/layout.d.ts delete mode 100644 types/metric.d.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 42f523a7..12b23124 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -1,21 +1,25 @@ import version from "../src/core/version"; -import { Event, IAugmentation, IDecodedEvent, IDecodedPayload, IPayload, Token } from "../types/data"; -import data from "./data"; -import diagnostic from "./diagnostic"; -import envelope from "./envelope"; -import interaction from "./interaction"; +import { Event, Payload, Token } from "../types/data"; +import { MetricEvent, PageEvent, PingEvent, SummaryEvent, TagEvent, UploadEvent } from "../types/decode/data"; +import { DecodedEvent, DecodedPayload } from "../types/decode/decode"; +import { BrokenImageEvent, ScriptErrorEvent } from "../types/decode/diagnostic"; +import { InputChangeEvent, PointerEvent, ResizeEvent, ScrollEvent } from "../types/decode/interaction"; +import { SelectionEvent, UnloadEvent, VisibilityEvent } from "../types/decode/interaction"; +import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, ResourceEvent } from "../types/decode/layout"; + +import * as data from "./data"; +import * as diagnostic from "./diagnostic"; +import * as interaction from "./interaction"; import * as layout from "./layout"; -import metric from "./metric"; import * as r from "./render"; -import * as summary from "./summary"; let pageId: string = null; -export function decode(input: string | IPayload, augmentations: IAugmentation = null): IDecodedPayload { - let json: IPayload = typeof input === "string" ? JSON.parse(input) : input; - let timestamp = augmentations ? augmentations.timestamp : Date.now(); - let ua = augmentations ? augmentations.ua : (navigator && "userAgent" in navigator ? navigator.userAgent : ""); - let payload: IDecodedPayload = { timestamp, ua, envelope: envelope(json.e), metrics: metric(json.m), analytics: [], playback: [] }; +export function decode(input: string): DecodedPayload { + let json: Payload = typeof input === "string" ? JSON.parse(input) : input; + let envelope = data.envelope(json.e); + let timestamp = Date.now(); + let payload: DecodedPayload = { timestamp, envelope }; let encoded: Token[][] = json.d; if (payload.envelope.version !== version) { @@ -23,18 +27,32 @@ export function decode(input: string | IPayload, augmentations: IAugmentation = } /* Reset components before decoding to keep them stateless */ - summary.reset(); + data.reset(); layout.reset(); for (let entry of encoded) { - let event: IDecodedEvent; - summary.decode(entry); + data.summarize(entry); switch (entry[1]) { - case Event.Scroll: - case Event.Document: - case Event.Resize: - case Event.Selection: - case Event.Change: + case Event.Page: + if (payload.page === undefined) { payload.page = []; } + payload.page.push(data.decode(entry) as PageEvent); + break; + case Event.Ping: + if (payload.ping === undefined) { payload.ping = []; } + payload.ping.push(data.decode(entry) as PingEvent); + break; + case Event.Tag: + if (payload.tag === undefined) { payload.tag = []; } + payload.tag.push(data.decode(entry) as TagEvent); + break; + case Event.Metric: + if (payload.metric === undefined) { payload.metric = []; } + payload.metric.push(data.decode(entry) as MetricEvent); + break; + case Event.Upload: + if (payload.upload === undefined) { payload.upload = []; } + payload.upload.push(data.decode(entry) as UploadEvent); + break; case Event.MouseDown: case Event.MouseUp: case Event.MouseMove: @@ -46,83 +64,117 @@ export function decode(input: string | IPayload, augmentations: IAugmentation = case Event.TouchCancel: case Event.TouchEnd: case Event.TouchMove: - event = interaction(entry); - payload.analytics.push(event); + if (payload.pointer === undefined) { payload.pointer = []; } + payload.pointer.push(interaction.decode(entry) as PointerEvent); + break; + case Event.Scroll: + if (payload.scroll === undefined) { payload.scroll = []; } + payload.scroll.push(interaction.decode(entry) as ScrollEvent); + break; + case Event.Resize: + if (payload.resize === undefined) { payload.resize = []; } + payload.resize.push(interaction.decode(entry) as ResizeEvent); + break; + case Event.Selection: + if (payload.selection === undefined) { payload.selection = []; } + payload.selection.push(interaction.decode(entry) as SelectionEvent); + break; + case Event.InputChange: + if (payload.input === undefined) { payload.input = []; } + payload.input.push(interaction.decode(entry) as InputChangeEvent); + break; + case Event.Unload: + if (payload.unload === undefined) { payload.unload = []; } + payload.unload.push(interaction.decode(entry) as UnloadEvent); + break; + case Event.Visibility: + if (payload.visibility === undefined) { payload.visibility = []; } + payload.visibility.push(interaction.decode(entry) as VisibilityEvent); break; case Event.BoxModel: - event = layout.decode(entry); - payload.playback.push(event); + if (payload.boxmodel === undefined) { payload.boxmodel = []; } + payload.boxmodel.push(layout.decode(entry) as BoxModelEvent); break; case Event.Discover: case Event.Mutation: - event = layout.decode(entry); - payload.playback.push(event); + if (payload.dom === undefined) { payload.dom = []; } + payload.dom.push(layout.decode(entry) as DomEvent); break; - case Event.Checksum: - event = layout.decode(entry); - payload.analytics.push(event); + case Event.Hash: + if (payload.hash === undefined) { payload.hash = []; } + payload.hash.push(layout.decode(entry) as HashEvent); break; - case Event.Page: - case Event.Ping: - case Event.Tag: - event = data(entry); - payload.analytics.push(event); + case Event.Document: + if (payload.doc === undefined) { payload.doc = []; } + payload.doc.push(layout.decode(entry) as DocumentEvent); break; case Event.ScriptError: + if (payload.script === undefined) { payload.script = []; } + payload.script.push(diagnostic.decode(entry) as ScriptErrorEvent); + break; case Event.ImageError: - event = diagnostic(entry); - payload.analytics.push(event); + if (payload.image === undefined) { payload.image = []; } + payload.image.push(diagnostic.decode(entry) as BrokenImageEvent); break; default: - event = {time: entry[0] as number, event: entry[1] as number, data: entry.slice(2)}; - payload.playback.push(event); + console.error(`No handler for Event: ${JSON.stringify(entry)}`); break; } } /* Enrich decoded payload with derived events */ - payload.analytics.push(...summary.enrich()); - payload.analytics.push(...layout.enrich()); + payload.summary = data.summary() as SummaryEvent[]; + if (layout.hashes.length > 0) { payload.hash = layout.hash() as HashEvent[]; } + if (layout.resources.length > 0) { payload.resource = layout.resource() as ResourceEvent[]; } return payload; } -export function html(decoded: IDecodedPayload): string { +export function html(decoded: DecodedPayload): string { let iframe = document.createElement("iframe"); render(decoded, iframe); return iframe.contentDocument.documentElement.outerHTML; } -export function render(decoded: IDecodedPayload, iframe: HTMLIFrameElement, header?: HTMLElement): void { +export function render(decoded: DecodedPayload, iframe: HTMLIFrameElement, header?: HTMLElement): void { // Reset rendering if we receive a new pageId if (pageId !== decoded.envelope.pageId) { pageId = decoded.envelope.pageId; r.reset(); } - // Render metrics - r.metric(decoded.metrics, header); - // Replay events - let events = [...decoded.analytics, ...decoded.playback].sort(sort); - replay(events, iframe); + let events: DecodedEvent[] = []; + for (let key in decoded) { + if (Array.isArray(decoded[key])) { + events = events.concat(decoded[key]); + } + } + replay(events.sort(sort), iframe, header); } -export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement): Promise { +export async function replay(events: DecodedEvent[], iframe: HTMLIFrameElement, header?: HTMLElement): Promise { let start = events[0].time; for (let entry of events) { if (entry.time - start > 16) { start = await wait(entry.time); } switch (entry.event) { + case Event.Page: + let pageEvent = entry as PageEvent; + r.page(pageEvent.data, iframe); + break; + case Event.Metric: + let metricEvent = entry as MetricEvent; + if (header) { r.metric(metricEvent.data, header); } + break; case Event.Discover: case Event.Mutation: - r.markup(entry.data, iframe); - break; - case Event.Checksum: - r.checksum(entry.data, iframe); + let domEvent = entry as DomEvent; + r.markup(domEvent.data, iframe); break; case Event.BoxModel: - r.boxmodel(entry.data, iframe); + let boxModelEvent = entry as BoxModelEvent; + r.boxmodel(boxModelEvent.data, iframe); break; case Event.MouseDown: case Event.MouseUp: @@ -131,25 +183,28 @@ export async function replay(events: IDecodedEvent[], iframe: HTMLIFrameElement) case Event.Click: case Event.DoubleClick: case Event.RightClick: - r.pointer(entry.event, entry.data, iframe); - break; case Event.TouchStart: case Event.TouchCancel: case Event.TouchEnd: case Event.TouchMove: - r.pointer(entry.event, entry.data, iframe); + let pointerEvent = entry as PointerEvent; + r.pointer(pointerEvent.event, pointerEvent.data, iframe); break; - case Event.Change: - r.change(entry.data, iframe); + case Event.InputChange: + let changeEvent = entry as InputChangeEvent; + r.change(changeEvent.data, iframe); break; case Event.Selection: - r.selection(entry.data, iframe); + let selectionEvent = entry as SelectionEvent; + r.selection(selectionEvent.data, iframe); break; case Event.Resize: - r.resize(entry.data, iframe); + let resizeEvent = entry as ResizeEvent; + r.resize(resizeEvent.data, iframe); break; case Event.Scroll: - r.scroll(entry.data, iframe); + let scrollEvent = entry as ScrollEvent; + r.scroll(scrollEvent.data, iframe); break; } } @@ -161,6 +216,6 @@ async function wait(timestamp: number): Promise { }); } -function sort(a: IDecodedEvent, b: IDecodedEvent): number { +function sort(a: DecodedEvent, b: DecodedEvent): number { return a.time - b.time; } diff --git a/decode/data.ts b/decode/data.ts index 5ffb7038..871dda8e 100644 --- a/decode/data.ts +++ b/decode/data.ts @@ -1,23 +1,79 @@ -import { Event, IDecodedEvent, IPageData, IPingData, ITagData, Token } from "../types/data"; +import { BooleanFlag, Envelope, Event, MetricData, PageData, PingData } from "../types/data"; +import { SummaryData, TagData, Token, Upload, UploadData } from "../types/data"; +import { DataEvent } from "../types/decode/data"; -export default function(tokens: Token[]): IDecodedEvent { +let summaries: { [key: number]: SummaryData[] } = null; +const SUMMARY_THRESHOLD = 250; + +export function reset(): void { + summaries = {}; +} + +export function decode(tokens: Token[]): DataEvent { let time = tokens[0] as number; let event = tokens[1] as Event; switch (event) { case Event.Page: - let page: IPageData = { + let page: PageData = { timestamp: tokens[2] as number, - elapsed: tokens[3] as number, + ua: tokens[3] as string, url: tokens[4] as string, - title: tokens[5] as string, - referrer: tokens[6] as string + referrer: tokens[5] as string, + lean: tokens[6] as BooleanFlag, }; return { time, event, data: page }; case Event.Ping: - let ping: IPingData = { gap: tokens[2] as number }; + let ping: PingData = { gap: tokens[2] as number }; return { time, event, data: ping }; case Event.Tag: - let tag: ITagData = { key: tokens[2] as string, value: tokens[3] as string }; + let tag: TagData = { key: tokens[2] as string, value: tokens[3] as string }; return { time, event, data: tag }; + case Event.Upload: + let upload: UploadData = { sequence: tokens[2] as number, attempts: tokens[3] as number, status: tokens[4] as number}; + return { time, event, data: upload }; + case Event.Metric: + let i = 0; + let metrics: MetricData = {}; + while (i < tokens.length) { + metrics[tokens[i++] as number] = tokens[i++] as number; + } + return { time, event, data: metrics }; + } +} + +export function envelope(tokens: Token[]): Envelope { + return { + sequence: tokens[0] as number, + version: tokens[1] as string, + projectId: tokens[2] as string, + userId: tokens[3] as string, + sessionId: tokens[4] as string, + pageId: tokens[5] as string, + upload: tokens[6] as Upload, + end: tokens[7] as BooleanFlag + }; +} + +export function summarize(entry: Token[]): void { + let time = entry[0] as number; + let type = entry[1] as Event; + let data: SummaryData = { event: type, start: time, end: time }; + if (!(type in summaries)) { summaries[type] = [data]; } + + let s = summaries[type][summaries[type].length - 1]; + if (time - s.end < SUMMARY_THRESHOLD) { s.end = time; } else { summaries[type].push(data); } +} + +export function summary(): DataEvent[] { + let data: SummaryData[] = []; + let time = null; + for (let type in summaries) { + if (summaries[type]) { + for (let d of summaries[type]) { + time = time ? Math.min(time, d.start) : d.start; + data.push(d); + } + } } + return data.length > 0 ? [{ time, event: Event.Summary, data }] : null; } diff --git a/decode/diagnostic.ts b/decode/diagnostic.ts index d26c9eeb..40e05d95 100644 --- a/decode/diagnostic.ts +++ b/decode/diagnostic.ts @@ -1,18 +1,19 @@ -import { Event, IDecodedEvent, Token } from "../types/data"; -import { IImageError, IScriptError } from "../types/diagnostic"; +import { Event, Token } from "../types/data"; +import { DiagnosticEvent } from "../types/decode/diagnostic"; +import { ImageErrorData, ScriptErrorData } from "../types/diagnostic"; -export default function(tokens: Token[]): IDecodedEvent { +export function decode(tokens: Token[]): DiagnosticEvent { let time = tokens[0] as number; let event = tokens[1] as Event; switch (event) { case Event.ImageError: - let imageError: IImageError = { + let imageError: ImageErrorData = { source: tokens[2] as string, target: tokens[3] as number }; return { time, event, data: imageError }; case Event.Selection: - let scriptError: IScriptError = { + let scriptError: ScriptErrorData = { source: tokens[2] as string, message: tokens[3] as string, line: tokens[4] as number, diff --git a/decode/envelope.ts b/decode/envelope.ts deleted file mode 100644 index ac27daf8..00000000 --- a/decode/envelope.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Flag, IEnvelope, Token, Upload } from "../types/data"; - -export default function(tokens: Token[]): IEnvelope { - return { - elapsed: tokens[0] as number, - sequence: tokens[1] as number, - version: tokens[2] as string, - projectId: tokens[3] as string, - userId: tokens[4] as string, - sessionId: tokens[5] as string, - pageId: tokens[6] as string, - upload: tokens[7] as Upload, - end: tokens[8] as Flag - }; -} diff --git a/decode/interaction.ts b/decode/interaction.ts index 9b4eb29b..f49ff96f 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -1,7 +1,8 @@ -import { Event, IDecodedEvent, Token } from "../types/data"; -import { IChange, IPointer, IResize, IScroll, ISelection } from "../types/interaction"; +import { Event, Token } from "../types/data"; +import { InteractionEvent } from "../types/decode/interaction"; +import { InputChangeData, PointerData, ResizeData, ScrollData, SelectionData, UnloadData, VisibilityData } from "../types/interaction"; -export default function(tokens: Token[]): IDecodedEvent { +export function decode(tokens: Token[]): InteractionEvent { let time = tokens[0] as number; let event = tokens[1] as Event; switch (event) { @@ -16,23 +17,23 @@ export default function(tokens: Token[]): IDecodedEvent { case Event.TouchCancel: case Event.TouchEnd: case Event.TouchMove: - let pointerData: IPointer = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; + let pointerData: PointerData = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; if (tokens.length > 6) { pointerData.targetX = tokens[5] as number; pointerData.targetY = tokens[6] as number; } return { time, event, data: pointerData }; case Event.Resize: - let resizeData: IResize = { width: tokens[2] as number, height: tokens[3] as number }; + let resizeData: ResizeData = { width: tokens[2] as number, height: tokens[3] as number }; return { time, event, data: resizeData }; - case Event.Change: - let changeData: IChange = { + case Event.InputChange: + let changeData: InputChangeData = { target: tokens[2] as number, value: tokens[3] as string }; return { time, event, data: changeData }; case Event.Selection: - let selectionData: ISelection = { + let selectionData: SelectionData = { start: tokens[2] as number, startOffset: tokens[3] as number, end: tokens[4] as number, @@ -40,9 +41,13 @@ export default function(tokens: Token[]): IDecodedEvent { }; return { time, event, data: selectionData }; case Event.Scroll: - let scrollData: IScroll = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; + let scrollData: ScrollData = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; return { time, event, data: scrollData }; - default: - return { time, event, data: tokens.slice(2) }; + case Event.Visibility: + let visibleData: VisibilityData = { visible: tokens[2] as string }; + return { time, event, data: visibleData }; + case Event.Unload: + let unloadData: UnloadData = { name: tokens[2] as string }; + return { time, event, data: unloadData }; } } diff --git a/decode/layout.ts b/decode/layout.ts index 1722f937..b42aafbe 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -1,58 +1,60 @@ -import hash from "../src/data/hash"; +import generateHash from "../src/data/hash"; import { resolve } from "../src/data/token"; -import { Event, IDecodedEvent, Token } from "../types/data"; -import { IAttributes, IBoxModel, IChecksum, IDecodedNode, IDocumentSize, ILayout, IResource } from "../types/layout"; +import { Event, Token } from "../types/data"; +import { DomData, LayoutEvent } from "../types/decode/layout"; +import { Attributes, BoxModelData, DocumentData, HashData, ResourceData } from "../types/layout"; const ID_ATTRIBUTE = "data-clarity"; let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; -let selectors: ILayout[]; -let resources: IResource[]; +export let hashes: HashData[]; +export let resources: ResourceData[]; let lastTime: number; export function reset(): void { - selectors = []; + hashes = []; resources = []; lastTime = null; } -export function decode(tokens: Token[]): IDecodedEvent { +export function decode(tokens: Token[]): LayoutEvent { let time = lastTime = tokens[0] as number; let event = tokens[1] as Event; - let decoded: IDecodedEvent = {time, event, data: []}; switch (event) { case Event.Document: - let d: IDocumentSize = { width: tokens[2] as number, height: tokens[3] as number }; - decoded.data.push(d); - return decoded; + let documentData: DocumentData = { width: tokens[2] as number, height: tokens[3] as number }; + return { time, event, data: documentData }; case Event.BoxModel: + let boxmodelData: BoxModelData[] = []; for (let i = 2; i < tokens.length; i += 2) { - let boxmodel: IBoxModel = { id: tokens[i] as number, box: tokens[i + 1] as number[] }; - decoded.data.push(boxmodel); + let boxmodel: BoxModelData = { id: tokens[i] as number, box: tokens[i + 1] as number[] }; + boxmodelData.push(boxmodel); } - return decoded; - case Event.Checksum: + return { time, event, data: boxmodelData }; + case Event.Hash: let reference = 0; + let hashData: HashData[] = []; for (let i = 2; i < tokens.length; i += 2) { let id = (tokens[i] as number) + reference; let token = tokens[i + 1]; - let checksum: IChecksum = { id, checksum: typeof(token) === "object" ? tokens[token[0]] : token }; - decoded.data.push(checksum); + let cs: HashData = { id, hash: typeof(token) === "object" ? tokens[token[0]] : token }; + hashData.push(cs); reference = id; } - return decoded; + return { time, event, data: hashData }; case Event.Discover: case Event.Mutation: let lastType = null; let node = []; let tagIndex = 0; + let domData: DomData[] = []; for (let i = 2; i < tokens.length; i++) { let token = tokens[i]; let type = typeof(token); switch (type) { case "number": if (type !== lastType && lastType !== null) { - decoded.data.push(process(node, tagIndex)); + domData.push(process(node, tagIndex)); node = []; tagIndex = 0; } @@ -81,20 +83,22 @@ export function decode(tokens: Token[]): IDecodedEvent { lastType = type; } // Process last node - decoded.data.push(process(node, tagIndex)); - return decoded; + domData.push(process(node, tagIndex)); + + return { time, event, data: domData }; } } -export function enrich(): IDecodedEvent[] { - let output = []; - if (selectors.length > 0) { output.push({ time: lastTime, event: Event.Layout, data: selectors }); } - if (resources.length > 0) { output.push({ time: lastTime, event: Event.Resource, data: resources }); } - return output; +export function hash(): LayoutEvent[] { + return hashes.length > 0 ? [{ time: lastTime, event: Event.Hash, data: hashes }] : null; +} + +export function resource(): LayoutEvent[] { + return resources.length > 0 ? [{ time: lastTime, event: Event.Resource, data: resources }] : null; } -function process(node: any[] | number[], tagIndex: number): IDecodedNode { - let output: IDecodedNode = { +function process(node: any[] | number[], tagIndex: number): DomData { + let output: DomData = { id: node[0], parent: tagIndex > 1 ? node[1] : null, next: tagIndex > 2 ? node[2] : null, @@ -103,7 +107,7 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { let hasAttribute = false; let attributes = {}; let value = null; - let path = output.parent in selectors ? `${selectors[output.parent]}>` : null; + let path = output.parent in hashes ? `${hashes[output.parent]}>` : null; for (let i = tagIndex + 1; i < node.length; i++) { let token = node[i] as string; @@ -130,15 +134,15 @@ function process(node: any[] | number[], tagIndex: number): IDecodedNode { } } - selector(output.id, output.tag, path, attributes); - resource(output.tag, attributes); + getHash(output.id, output.tag, path, attributes); + getResource(output.tag, attributes); if (hasAttribute) { output.attributes = attributes; } if (value) { output.value = value; } return output; } -function selector(id: number, tag: string, path: string, attributes: IAttributes): void { +function getHash(id: number, tag: string, path: string, attributes: Attributes): void { switch (tag) { case "STYLE": case "TITLE": @@ -152,12 +156,12 @@ function selector(id: number, tag: string, path: string, attributes: IAttributes if ("id" in attributes) { s = `${tag}#${attributes["id"]}`; } if ("class" in attributes) { s += `.${attributes["class"].trim().split(" ").join(".")}`; } if (ID_ATTRIBUTE in attributes) { s = `*${attributes[ID_ATTRIBUTE]}`; } - if (s) { selectors.push({ id, checksum: hash(s), selector: s }); } + if (s) { hashes.push({ id, hash: generateHash(s), selector: s }); } break; } } -function resource(tag: string, attributes: IAttributes): void { +function getResource(tag: string, attributes: Attributes): void { switch (tag) { case "LINK": if ("href" in attributes && "rel" in attributes && attributes["rel"] === "stylesheet") { diff --git a/decode/metric.ts b/decode/metric.ts deleted file mode 100644 index 16ae94ba..00000000 --- a/decode/metric.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Token } from "../types/data"; -import { IMetric } from "../types/metric"; - -let metrics: IMetric = null; - -export default function(tokens: Token[]): IMetric { - let i = 0; - metrics = {}; - while (i < tokens.length) { - metrics[tokens[i++] as number] = tokens[i++] as number; - } - return metrics; -} diff --git a/decode/render.ts b/decode/render.ts index 69016858..4f28a7cd 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,11 +1,11 @@ -import { Event } from "../types/data"; -import { IChange, IPointer, IResize, IScroll, ISelection } from "../types/interaction"; -import { IBoxModel, IChecksum, IDecodedNode } from "../types/layout"; -import { IMetric, Metric } from "../types/metric"; +import { Event, Metric, MetricData, PageData } from "../types/data"; +import { DomData } from "../types/decode/layout"; +import { InputChangeData, PointerData, ResizeData, ScrollData, SelectionData } from "../types/interaction"; +import { BoxModelData } from "../types/layout"; let nodes = {}; let boxmodels = {}; -let metrics: IMetric = null; +let metrics: MetricData = null; let svgns: string = "http://www.w3.org/2000/svg"; let lean = false; const METRIC_MAP = {}; @@ -44,7 +44,7 @@ export function reset(): void { metrics = {}; } -export function metric(data: IMetric, header: HTMLElement): void { +export function metric(data: MetricData, header: HTMLElement): void { let html = []; // Copy over metrics for future reference @@ -70,11 +70,11 @@ function value(input: number, unit: string): number { } } -export function checksum(data: IChecksum[], iframe: HTMLIFrameElement): void { - lean = true; +export function page(data: PageData, iframe: HTMLIFrameElement): void { + lean = !!data.lean; } -export function boxmodel(data: IBoxModel[], iframe: HTMLIFrameElement): void { +export function boxmodel(data: BoxModelData[], iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; for (let bm of data) { let el = element(bm.id) as HTMLElement; @@ -108,7 +108,7 @@ function css(style: CSSStyleDeclaration, field: string): number { return parseInt(style[field], 10); } -export function markup(data: IDecodedNode[], iframe: HTMLIFrameElement): void { +export function markup(data: DomData[], iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; for (let node of data) { let parent = element(node.parent); @@ -189,7 +189,7 @@ function element(nodeId: number): Node { return nodeId !== null && nodeId > 0 && nodeId in nodes ? nodes[nodeId] : null; } -function insert(data: IDecodedNode, parent: Node, node: Node, next: Node): void { +function insert(data: DomData, parent: Node, node: Node, next: Node): void { if (parent !== null) { next = next && next.parentElement !== parent ? null : next; try { @@ -232,12 +232,12 @@ function setAttributes(node: HTMLElement, attributes: object): void { } } -export function scroll(data: IScroll, iframe: HTMLIFrameElement): void { +export function scroll(data: ScrollData, iframe: HTMLIFrameElement): void { let target = getNode(data.target); if (target) { target.scrollTo(data.x, data.y); } } -export function resize(data: IResize, iframe: HTMLIFrameElement): void { +export function resize(data: ResizeData, iframe: HTMLIFrameElement): void { iframe.removeAttribute("style"); let margin = 10; let px = "px"; @@ -258,18 +258,18 @@ export function resize(data: IResize, iframe: HTMLIFrameElement): void { iframe.style.overflow = "hidden"; } -export function change(data: IChange, iframe: HTMLIFrameElement): void { +export function change(data: InputChangeData, iframe: HTMLIFrameElement): void { let el = element(data.target) as HTMLInputElement; if (el) { el.value = data.value; } } -export function selection(data: ISelection, iframe: HTMLIFrameElement): void { +export function selection(data: SelectionData, iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; let s = doc.getSelection(); s.setBaseAndExtent(element(data.start), data.startOffset, element(data.end), data.endOffset); } -export function pointer(event: Event, data: IPointer, iframe: HTMLIFrameElement): void { +export function pointer(event: Event, data: PointerData, iframe: HTMLIFrameElement): void { let doc = iframe.contentDocument; let p = doc.getElementById("clarity-pointer"); let pointerWidth = 20; diff --git a/decode/summary.ts b/decode/summary.ts deleted file mode 100644 index 5377f27f..00000000 --- a/decode/summary.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Event, IDecodedEvent, IEventSummary, Token } from "../types/data"; - -let summary: { [key: number]: IEventSummary[] } = null; -const SUMMARY_THRESHOLD = 250; - -export function reset(): void { - summary = {}; -} - -export function decode(entry: Token[]): void { - let time = entry[0] as number; - let type = entry[1] as Event; - let data: IEventSummary = { event: type, start: time, end: time }; - if (!(type in summary)) { summary[type] = [data]; } - - let s = summary[type][summary[type].length - 1]; - if (time - s.end < SUMMARY_THRESHOLD) { s.end = time; } else { summary[type].push(data); } -} - -export function enrich(): IDecodedEvent[] { - let data: IEventSummary[] = []; - let time = null; - for (let type in summary) { - if (summary[type]) { - for (let d of summary[type]) { - time = time ? Math.min(time, d.start) : d.start; - data.push(d); - } - } - } - return data.length > 0 ? [{ time, event: Event.Summary, data }] : []; -} diff --git a/package.json b/package.json index 770123fc..84a0d3d6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.5", + "version": "1.0.0-beta.6", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/clarity.ts b/src/clarity.ts index 3eb0a4e7..1cff76cb 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -1,4 +1,4 @@ -import { IConfig } from "@clarity-types/core"; +import { Config } from "@clarity-types/core"; import * as core from "@src/core"; import configuration from "@src/core/config"; import { bind } from "@src/core/event"; @@ -6,12 +6,11 @@ import * as data from "@src/data"; import * as diagnostic from "@src/diagnostic"; import * as interaction from "@src/interaction"; import * as layout from "@src/layout"; -import * as metric from "@src/metric"; export { tag } from "@src/data/tag"; let status = false; -export function config(override: IConfig): boolean { +export function config(override: Config): boolean { // Process custom configuration overrides, if available if (status) { return false; } for (let key in override) { @@ -20,13 +19,12 @@ export function config(override: IConfig): boolean { return true; } -export function start(override: IConfig = {}): void { +export function start(override: Config = {}): void { if (core.check()) { config(override); status = true; core.start(); - metric.start(); data.start(); diagnostic.start(); layout.start(); @@ -53,7 +51,6 @@ export function end(): void { layout.end(); diagnostic.end(); data.end(); - metric.end(); core.end(); status = false; diff --git a/src/core/config.ts b/src/core/config.ts index 6d2811df..8b673f47 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,6 +1,6 @@ -import { IConfig } from "@clarity-types/core"; +import { Config } from "@clarity-types/core"; -let config: IConfig = { +let config: Config = { projectId: null, longtask: 30, lookahead: 500, diff --git a/src/core/event.ts b/src/core/event.ts index 769803ef..9d012e13 100644 --- a/src/core/event.ts +++ b/src/core/event.ts @@ -1,6 +1,6 @@ -import { IEventBindingData } from "@clarity-types/core"; +import { BrowserEvent } from "@clarity-types/core"; -let bindings: IEventBindingData[] = []; +let bindings: BrowserEvent[] = []; export function bind(target: EventTarget, event: string, listener: EventListener, capture: boolean = false): void { target.addEventListener(event, listener, capture); diff --git a/src/core/task.ts b/src/core/task.ts index 242ac54c..053dad1e 100644 --- a/src/core/task.ts +++ b/src/core/task.ts @@ -1,12 +1,12 @@ -import { IAsyncTask, ITaskTracker, TaskFunction, TaskResolve } from "@clarity-types/core"; -import { Metric } from "@clarity-types/metric"; +import { AsyncTask, TaskFunction, TaskResolve, TaskTiming } from "@clarity-types/core"; +import { Metric } from "@clarity-types/data"; import config from "@src/core/config"; -import * as metrics from "@src/metric"; +import * as metrics from "@src/data/metric"; -let tracker: ITaskTracker = {}; +let tracker: TaskTiming = {}; let threshold = config.longtask; -let queue: IAsyncTask[] = []; -let active: IAsyncTask = null; +let queue: AsyncTask[] = []; +let active: AsyncTask = null; export async function schedule(task: TaskFunction): Promise { // If this task is already scheduled, skip it diff --git a/src/core/version.ts b/src/core/version.ts index d6fbc747..5e2d6bb9 100644 --- a/src/core/version.ts +++ b/src/core/version.ts @@ -1,2 +1,2 @@ -let version = "1.0.0-b5"; +let version = "1.0.0-b6"; export default version; diff --git a/src/data/encode.ts b/src/data/encode.ts index 63a5a42c..9d7476f6 100644 --- a/src/data/encode.ts +++ b/src/data/encode.ts @@ -1,11 +1,10 @@ -import {Event, Token } from "@clarity-types/data"; -import { Metric } from "@clarity-types/metric"; +import {Event, Metric, Token } from "@clarity-types/data"; import time from "@src/core/time"; import { metadata } from "@src/data/metadata"; +import * as metric from "@src/data/metric"; import * as ping from "@src/data/ping"; import * as tag from "@src/data/tag"; -import * as metric from "@src/metric"; -import { queue } from "./upload"; +import { queue, track } from "./upload"; export default function(event: Event): void { let t = time(); @@ -18,10 +17,10 @@ export default function(event: Event): void { case Event.Page: metric.counter(Metric.StartTime, Math.round(performance.now())); tokens.push(metadata.page.timestamp); - tokens.push(metadata.page.elapsed); + tokens.push(metadata.page.ua); tokens.push(metadata.page.url); - tokens.push(metadata.page.title); tokens.push(metadata.page.referrer); + tokens.push(metadata.page.lean); queue(tokens); break; case Event.Tag: @@ -29,5 +28,26 @@ export default function(event: Event): void { tokens.push(tag.data.value); queue(tokens); break; + case Event.Upload: + tokens.push(track.sequence); + tokens.push(track.attempts); + tokens.push(track.status); + queue(tokens); + break; + case Event.Metric: + if (metric.updates.length > 0) { + for (let d in metric.data) { + if (metric.data[d]) { + let m = parseInt(d, 10); + if (metric.updates.indexOf(m) >= 0) { + tokens.push(m); + tokens.push(metric.data[d]); + } + } + } + metric.reset(); + queue(tokens); + } + break; } } diff --git a/src/data/index.ts b/src/data/index.ts index 296fa189..13dcf31f 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -1,10 +1,12 @@ import * as metadata from "@src/data/metadata"; +import * as metric from "@src/data/metric"; import * as ping from "@src/data/ping"; import * as tag from "@src/data/tag"; import * as upload from "@src/data/upload"; export function start(): void { upload.start(); + metric.start(); metadata.start(); ping.start(); tag.reset(); @@ -15,4 +17,5 @@ export function end(): void { ping.end(); upload.end(); metadata.end(); + metric.end(); } diff --git a/src/data/metadata.ts b/src/data/metadata.ts index cce09572..3a8545d1 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -1,6 +1,5 @@ -import { Event, Flag, ICookieData, IEnvelope, IMetadata, IPageData, Token, Upload } from "@clarity-types/data"; +import { BooleanFlag, CookieInfo, Envelope, Event, Metadata, PageData, Token, Upload } from "@clarity-types/data"; import config from "@src/core/config"; -import time from "@src/core/time"; import version from "@src/core/version"; import encode from "@src/data/encode"; import hash from "@src/data/hash"; @@ -8,18 +7,20 @@ import hash from "@src/data/hash"; const CLARITY_COOKIE_NAME: string = "_clarity"; const CLARITY_COOKIE_SEPARATOR: string = "|"; const CLARITY_SESSION_LENGTH = 30 * 60 * 1000; -export let metadata: IMetadata = null; +export let metadata: Metadata = null; export function start(): void { - let cookie: ICookieData = read(); + let cookie: CookieInfo = read(); let ts = Date.now(); - let elapsed = time(); let projectId = config.projectId || hash(location.host); let userId = cookie && cookie.userId ? cookie.userId : guid(); let sessionId = cookie && cookie.sessionId && ts - cookie.timestamp < CLARITY_SESSION_LENGTH ? cookie.sessionId : ts.toString(36); let pageId = guid(); - let e: IEnvelope = { elapsed, sequence: 0, version, pageId, userId, sessionId, projectId, upload: Upload.Async, end: Flag.False }; - let p: IPageData = { timestamp: ts, elapsed, url: location.href, title: document.title, referrer: document.referrer }; + let ua = navigator && "userAgent" in navigator ? navigator.userAgent : ""; + let upload = Upload.Async; + let lean = config.lean ? BooleanFlag.True : BooleanFlag.False; + let e: Envelope = { sequence: 0, version, pageId, userId, sessionId, projectId, upload, end: BooleanFlag.False }; + let p: PageData = { timestamp: ts, ua, url: location.href, referrer: document.referrer, lean }; metadata = { page: p, envelope: e }; @@ -35,13 +36,10 @@ export function end(): void { export function envelope(last: boolean, backup: boolean = false): Token[] { let e = metadata.envelope; e.upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); - e.end = last ? Flag.True : Flag.False; - if (e.upload !== Upload.Backup) { - e.elapsed = time(); - e.sequence++; - } + e.end = last ? BooleanFlag.True : BooleanFlag.False; + if (e.upload !== Upload.Backup) { e.sequence++; } - return [e.elapsed, e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, e.upload, e.end]; + return [e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, e.upload, e.end]; } // Credit: http://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript @@ -62,7 +60,7 @@ function guid() { } // tslint:enable -function track(data: ICookieData): void { +function track(data: CookieInfo): void { let expiry = new Date(); expiry.setDate(expiry.getDate() + config.expire); let expires = expiry ? "expires=" + expiry.toUTCString() : ""; @@ -70,7 +68,7 @@ function track(data: ICookieData): void { document.cookie = CLARITY_COOKIE_NAME + "=" + value; } -function read(): ICookieData { +function read(): CookieInfo { let cookies: string[] = document.cookie.split(";"); if (cookies) { for (let i = 0; i < cookies.length; i++) { diff --git a/src/metric/index.ts b/src/data/metric.ts similarity index 52% rename from src/metric/index.ts rename to src/data/metric.ts index f2c521bc..a9d430b3 100644 --- a/src/metric/index.ts +++ b/src/data/metric.ts @@ -1,28 +1,33 @@ -import { IMetric, Metric } from "@clarity-types/metric"; +import { Event, Metric, MetricData } from "@clarity-types/data"; +import encode from "./encode"; -export let metrics: IMetric = null; +export let data: MetricData = null; export let updates: Metric[] = []; export function start(): void { - metrics = {}; + data = {}; } export function end(): void { - metrics = null; + data = null; } export function counter(metric: Metric, increment: number = 1): void { - if (!(metric in metrics)) { metrics[metric] = 0; } - metrics[metric] += increment; + if (!(metric in data)) { data[metric] = 0; } + data[metric] += increment; track(metric); } export function measure(metric: Metric, value: number): void { - if (!(metric in metrics)) { metrics[metric] = 0; } - metrics[metric] = Math.max(value, metrics[metric]); + if (!(metric in data)) { data[metric] = 0; } + data[metric] = Math.max(value, data[metric]); track(metric); } +export function compute(): void { + encode(Event.Metric); +} + function track(metric: Metric): void { if (updates.indexOf(metric) === -1) { updates.push(metric); diff --git a/src/data/ping.ts b/src/data/ping.ts index f57f5695..3611cef3 100644 --- a/src/data/ping.ts +++ b/src/data/ping.ts @@ -1,10 +1,10 @@ -import { Event, IPingData } from "@clarity-types/data"; +import { Event, PingData } from "@clarity-types/data"; import { pause } from "@src/clarity"; import config from "@src/core/config"; import time from "@src/core/time"; import encode from "./encode"; -export let data: IPingData; +export let data: PingData; let last = 0; let interval = 0; let timeout: number = null; diff --git a/src/data/tag.ts b/src/data/tag.ts index 9d2d5e6f..9b974715 100644 --- a/src/data/tag.ts +++ b/src/data/tag.ts @@ -1,7 +1,7 @@ -import { Event, ITagData } from "@clarity-types/data"; +import { Event, TagData } from "@clarity-types/data"; import encode from "@src/data/encode"; -export let data: ITagData = null; +export let data: TagData = null; export function reset(): void { data = null; diff --git a/src/data/upload.ts b/src/data/upload.ts index ea14bd61..000b699e 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,16 +1,20 @@ -import { Event, IEncodedPayload, Token } from "@clarity-types/data"; -import { Metric } from "@clarity-types/metric"; +import { EncodedPayload, Event, Metric, Token, Transit, UploadData } from "@clarity-types/data"; import config from "@src/core/config"; -import {envelope} from "@src/data/metadata"; +import encode from "@src/data/encode"; +import { envelope, metadata } from "@src/data/metadata"; +import * as metric from "@src/data/metric"; import * as ping from "@src/data/ping"; -import { counter } from "@src/metric"; -import metrics from "@src/metric/encode"; +const MAX_RETRIES = 2; let events: string[]; let timeout: number = null; +let transit: Transit; +export let track: UploadData; export function start(): void { events = []; + transit = {}; + track = null; recover(); } @@ -20,23 +24,26 @@ export function queue(data: Token[]): void { events.push(event); switch (type) { + case Event.Metric: + case Event.Upload: + return; // do not schedule upload callback case Event.Discover: case Event.Mutation: case Event.BoxModel: - case Event.Checksum: + case Event.Hash: case Event.Document: - counter(Metric.LayoutBytes, event.length); + metric.counter(Metric.LayoutBytes, event.length); break; case Event.Network: case Event.Performance: - counter(Metric.NetworkBytes, event.length); + metric.counter(Metric.NetworkBytes, event.length); break; case Event.ScriptError: case Event.ImageError: - counter(Metric.DiagnosticBytes, event.length); + metric.counter(Metric.DiagnosticBytes, event.length); break; default: - counter(Metric.InteractionBytes, event.length); + metric.counter(Metric.InteractionBytes, event.length); break; } @@ -48,33 +55,51 @@ export function end(): void { clearTimeout(timeout); upload(true); events = []; + transit = {}; + track = null; } function upload(last: boolean = false): void { + metric.compute(); let handler = config.upload ? config.upload : send; - let payload: IEncodedPayload = {e: JSON.stringify(envelope(last)), m: JSON.stringify(metrics(last)), d: `[${events.join()}]`}; - handler(stringify(payload), last); - if (last) { backup(payload); } + let payload: EncodedPayload = {e: JSON.stringify(envelope(last)), d: `[${events.join()}]`}; + let sequence = metadata.envelope.sequence; + transit[sequence] = { data: stringify(payload), attempts: 1 }; + handler(transit[sequence].data, sequence, last); + if (last) { backup(payload); } else { ping.reset(); } events = []; - if (!last) { ping.reset(); } } -function stringify(payload: IEncodedPayload): string { - return `{"e":${payload.e},"m":${payload.m},"d":${payload.d}}`; +function stringify(payload: EncodedPayload): string { + return `{"e":${payload.e},"d":${payload.d}}`; } -function send(data: string, last: boolean = false): void { +function send(data: string, sequence: number = null, last: boolean = false): void { if (config.url.length > 0) { if (last && "sendBeacon" in navigator) { navigator.sendBeacon(config.url, data); } else { let xhr = new XMLHttpRequest(); xhr.open("POST", config.url); + if (sequence !== null) { xhr.onreadystatechange = (): void => { check(xhr, sequence); }; } xhr.send(data); } } } +function check(xhr: XMLHttpRequest, sequence: number): void { + if (xhr.readyState === XMLHttpRequest.DONE) { + if ((xhr.status < 200 || xhr.status > 208) && transit[sequence].attempts <= MAX_RETRIES) { + transit[sequence].attempts++; + send(transit[sequence].data, sequence); + } else { + track = { sequence, attempts: transit[sequence].attempts, status: xhr.status }; + encode(Event.Upload); + delete transit[sequence]; + } + } +} + function recover(): void { if (typeof localStorage !== "undefined") { let data = localStorage.getItem("clarity-backup"); @@ -84,7 +109,7 @@ function recover(): void { } } -function backup(payload: IEncodedPayload): void { +function backup(payload: EncodedPayload): void { if (typeof localStorage !== "undefined") { payload.e = JSON.stringify(envelope(true, true)); localStorage.setItem("clarity-backup", stringify(payload)); diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts index 8dabd11e..a06d625d 100644 --- a/src/diagnostic/encode.ts +++ b/src/diagnostic/encode.ts @@ -1,10 +1,9 @@ -import {Event, Token} from "@clarity-types/data"; -import {Metric} from "@clarity-types/metric"; +import {Event, Metric, Token} from "@clarity-types/data"; import time from "@src/core/time"; +import * as metric from "@src/data/metric"; import { queue } from "@src/data/upload"; import * as image from "@src/diagnostic/image"; import * as script from "@src/diagnostic/script"; -import * as metric from "@src/metric"; export default function(type: Event): Token[] { let tokens: Token[] = [time(), type]; diff --git a/src/diagnostic/image.ts b/src/diagnostic/image.ts index 64753320..34324539 100644 --- a/src/diagnostic/image.ts +++ b/src/diagnostic/image.ts @@ -1,10 +1,10 @@ import { Event } from "@clarity-types/data"; -import { IImageError } from "@clarity-types/diagnostic"; +import { ImageErrorData } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -export let data: IImageError[] = []; +export let data: ImageErrorData[] = []; export function start(): void { bind(document, "error", handler, true); diff --git a/src/diagnostic/script.ts b/src/diagnostic/script.ts index 69e13f1a..7754c552 100644 --- a/src/diagnostic/script.ts +++ b/src/diagnostic/script.ts @@ -1,9 +1,9 @@ import { Event } from "@clarity-types/data"; -import { IScriptError } from "@clarity-types/diagnostic"; +import { ScriptErrorData } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; import encode from "./encode"; -export let data: IScriptError[] = []; +export let data: ScriptErrorData[] = []; export function start(): void { bind(window, "error", handler); diff --git a/src/interaction/change.ts b/src/interaction/change.ts index 954ec374..0e0d5fca 100644 --- a/src/interaction/change.ts +++ b/src/interaction/change.ts @@ -1,11 +1,11 @@ import { Event } from "@clarity-types/data"; -import { IChange } from "@clarity-types/interaction"; +import { InputChangeData } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; import mask from "@src/core/mask"; import { get } from "@src/layout/dom"; import encode from "./encode"; -export let data: IChange; +export let data: InputChangeData; export function start(): void { bind(document, "change", recompute, true); @@ -16,7 +16,7 @@ function recompute(evt: UIEvent): void { let value = get(target); if (target && value) { data = { target: value.id, value: value.metadata.masked ? mask(target.value) : target.value }; - encode(Event.Change); + encode(Event.InputChange); } } diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 91180a79..800c9b53 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -1,8 +1,7 @@ -import {Event, Token} from "@clarity-types/data"; -import {Metric} from "@clarity-types/metric"; +import {Event, Metric, Token} from "@clarity-types/data"; import time from "@src/core/time"; +import * as metric from "@src/data/metric"; import { queue } from "@src/data/upload"; -import * as metric from "@src/metric"; import * as change from "./change"; import * as pointer from "./pointer"; import * as resize from "./resize"; @@ -56,7 +55,7 @@ export default function(type: Event): void { metric.counter(Metric.EndTime, t); unload.reset(); break; - case Event.Change: + case Event.InputChange: let ch = change.data; tokens.push(ch.target); tokens.push(ch.value); diff --git a/src/interaction/pointer.ts b/src/interaction/pointer.ts index 79df4986..e2c30208 100644 --- a/src/interaction/pointer.ts +++ b/src/interaction/pointer.ts @@ -1,5 +1,5 @@ import { Event } from "@clarity-types/data"; -import { IPointer } from "@clarity-types/interaction"; +import { PointerData } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; @@ -7,7 +7,7 @@ import * as boxmodel from "@src/layout/boxmodel"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -export let data: { [key: number]: IPointer[] } = {}; +export let data: { [key: number]: PointerData[] } = {}; let timeout: number = null; export function start(): void { @@ -56,7 +56,7 @@ function touch(event: Event, evt: TouchEvent): void { } } -function handler(event: Event, current: IPointer): void { +function handler(event: Event, current: PointerData): void { switch (event) { case Event.MouseMove: case Event.MouseWheel: @@ -78,7 +78,7 @@ function handler(event: Event, current: IPointer): void { export function reset(): void { data = {}; - let mouseEvents = [Event.MouseDown, Event.MouseUp, Event.MouseWheel, Event.MouseMove, Event.DoubleClick, Event.Click]; + let mouseEvents = [Event.MouseDown, Event.MouseUp, Event.MouseWheel, Event.MouseMove, Event.DoubleClick, Event.Click, Event.RightClick]; let touchEvents = [Event.TouchStart, Event.TouchMove, Event.TouchEnd, Event.TouchCancel]; let events = mouseEvents.concat(touchEvents); for (let event of events) { @@ -86,7 +86,7 @@ export function reset(): void { } } -function similar(last: IPointer, current: IPointer): boolean { +function similar(last: PointerData, current: PointerData): boolean { let dx = last.x - current.x; let dy = last.y - current.y; let distance = Math.sqrt(dx * dx + dy * dy); diff --git a/src/interaction/resize.ts b/src/interaction/resize.ts index 63323842..f1a3cc82 100644 --- a/src/interaction/resize.ts +++ b/src/interaction/resize.ts @@ -1,9 +1,9 @@ import { Event } from "@clarity-types/data"; -import { IResize } from "@clarity-types/interaction"; +import { ResizeData } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; import encode from "./encode"; -export let data: IResize; +export let data: ResizeData; export function start(): void { bind(window, "resize", recompute); diff --git a/src/interaction/scroll.ts b/src/interaction/scroll.ts index 185f0230..edd522bd 100644 --- a/src/interaction/scroll.ts +++ b/src/interaction/scroll.ts @@ -1,12 +1,12 @@ import { Event } from "@clarity-types/data"; -import { IScroll } from "@clarity-types/interaction"; +import { ScrollData } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -export let data: IScroll[] = []; +export let data: ScrollData[] = []; let timeout: number = null; export function start(): void { @@ -16,9 +16,9 @@ export function start(): void { function recompute(event: UIEvent = null): void { let eventTarget = event ? (event.target === document ? document.documentElement : event.target) : document.documentElement; - let x = (eventTarget as HTMLElement).scrollLeft; - let y = (eventTarget as HTMLElement).scrollTop; - let current: IScroll = {target: getId(eventTarget as Node), x, y, time: time()}; + let x = Math.round((eventTarget as HTMLElement).scrollLeft); + let y = Math.round((eventTarget as HTMLElement).scrollTop); + let current: ScrollData = {target: getId(eventTarget as Node), x, y, time: time()}; let length = data.length; let last = length > 1 ? data[length - 2] : null; @@ -33,7 +33,7 @@ export function reset(): void { data = []; } -function similar(last: IScroll, current: IScroll): boolean { +function similar(last: ScrollData, current: ScrollData): boolean { let dx = last.x - current.x; let dy = last.y - current.y; return (dx * dx + dy * dy < config.distance * config.distance) && (current.time - last.time < config.interval); diff --git a/src/interaction/selection.ts b/src/interaction/selection.ts index 25372b3d..5dadf2e2 100644 --- a/src/interaction/selection.ts +++ b/src/interaction/selection.ts @@ -1,11 +1,11 @@ import { Event } from "@clarity-types/data"; -import { ISelection } from "@clarity-types/interaction"; +import { SelectionData } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -export let data: ISelection = null; +export let data: SelectionData = null; let selection: Selection = null; let timeout: number = null; diff --git a/src/interaction/unload.ts b/src/interaction/unload.ts index 846af2c4..c632f490 100644 --- a/src/interaction/unload.ts +++ b/src/interaction/unload.ts @@ -1,10 +1,10 @@ import { Event } from "@clarity-types/data"; -import { IUnload } from "@clarity-types/interaction"; +import { UnloadData } from "@clarity-types/interaction"; import { end } from "@src/clarity"; import { bind } from "@src/core/event"; import encode from "./encode"; -export let data: IUnload; +export let data: UnloadData; export function start(): void { bind(window, "beforeunload", recompute); diff --git a/src/interaction/visibility.ts b/src/interaction/visibility.ts index f27b4a50..94ea0091 100644 --- a/src/interaction/visibility.ts +++ b/src/interaction/visibility.ts @@ -1,9 +1,9 @@ import { Event } from "@clarity-types/data"; -import { IVisibility } from "@clarity-types/interaction"; +import { VisibilityData } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; import encode from "./encode"; -export let data: IVisibility; +export let data: VisibilityData; export function start(): void { bind(document, "visibilitychange", recompute); diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index ef9f3b8b..5fa359f2 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -1,12 +1,11 @@ -import { Event } from "@clarity-types/data"; -import { IBoxModel } from "@clarity-types/layout"; -import { Metric } from "@clarity-types/metric"; +import { Event, Metric } from "@clarity-types/data"; +import { BoxModelData } from "@clarity-types/layout"; import config from "@src/core/config"; import * as task from "@src/core/task"; import encode from "@src/layout/encode"; import * as dom from "./dom"; -let bm: {[key: number]: IBoxModel} = {}; +let bm: {[key: number]: BoxModelData} = {}; let updateMap: number[] = []; let timeout: number = null; @@ -40,7 +39,7 @@ async function boxmodel(): Promise { task.stop(timer); } -export function updates(): IBoxModel[] { +export function updates(): BoxModelData[] { let summary = []; for (let id of updateMap) { summary.push(bm[id]); @@ -58,7 +57,7 @@ export function relative(x: number, y: number, element: Element): number[] { } function update(id: number, box: number[]): void { - let changed = true; + let changed = box !== null; if (id in bm) { changed = box.length === bm[id].box.length ? false : true; if (changed === false) { @@ -78,7 +77,7 @@ function update(id: number, box: number[]): void { } function layout(element: Element, x: number = 0, y: number = 0): number[] { - let box: number[] = [0, 0, 0, 0]; + let box: number[] = null; let rect = element.getBoundingClientRect(); if (rect && rect.width > 0 && rect.height > 0) { diff --git a/src/layout/discover.ts b/src/layout/discover.ts index 0f174447..f42e5d52 100644 --- a/src/layout/discover.ts +++ b/src/layout/discover.ts @@ -1,6 +1,5 @@ -import { Event } from "@clarity-types/data"; +import { Event, Metric } from "@clarity-types/data"; import { Source } from "@clarity-types/layout"; -import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as task from "@src/core/task"; import * as boxmodel from "@src/layout/boxmodel"; @@ -26,6 +25,6 @@ async function discover(): Promise { processNode(node, Source.Discover); node = walker.nextNode(); } - await encode(config.lean ? Event.Checksum : Event.Discover); + await encode(config.lean ? Event.Hash : Event.Discover); task.stop(timer); } diff --git a/src/layout/document.ts b/src/layout/document.ts index 619866ed..cca59cec 100644 --- a/src/layout/document.ts +++ b/src/layout/document.ts @@ -1,25 +1,28 @@ import { Event } from "@clarity-types/data"; -import { IDocumentSize } from "@clarity-types/layout"; +import { DocumentData } from "@clarity-types/layout"; import encode from "./encode"; -export let doc: IDocumentSize; +export let data: DocumentData; + +export function reset(): void { + data = null; +} export function compute(): void { let body = document.body; let d = document.documentElement; + let width = body ? body.clientWidth : null; let bodyClientHeight = body ? body.clientHeight : null; let bodyScrollHeight = body ? body.scrollHeight : null; let bodyOffsetHeight = body ? body.offsetHeight : null; let documentClientHeight = d ? d.clientHeight : null; let documentScrollHeight = d ? d.scrollHeight : null; let documentOffsetHeight = d ? d.offsetHeight : null; - let documentHeight = Math.max(bodyClientHeight, bodyScrollHeight, bodyOffsetHeight, + let height = Math.max(bodyClientHeight, bodyScrollHeight, bodyOffsetHeight, documentClientHeight, documentScrollHeight, documentOffsetHeight); - doc = { - width: body ? body.clientWidth : null, - height: documentHeight - }; - - encode(Event.Document); + if (data === null || width !== data.width || height !== data.height) { + data = { width, height }; + encode(Event.Document); + } } diff --git a/src/layout/dom.ts b/src/layout/dom.ts index c6e83bc2..33eeb49c 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -1,4 +1,4 @@ -import { INodeChange, INodeData, INodeValue, Source } from "@clarity-types/layout"; +import { NodeChange, NodeInfo, NodeValue, Source } from "@clarity-types/layout"; import time from "@src/core/time"; const NODE_ID_PROP: string = "__node_index__"; @@ -9,8 +9,8 @@ const UNMASK_ATTRIBUTE = "data-clarity-unmask"; let index: number = 1; let nodes: Node[] = []; -let values: INodeValue[] = []; -let changes: INodeChange[][] = []; +let values: NodeValue[] = []; +let changes: NodeChange[][] = []; let updateMap: number[] = []; let selectorMap: number[] = []; @@ -33,7 +33,7 @@ export function getId(node: Node, autogen: boolean = false): number { return id ? id : null; } -export function add(node: Node, data: INodeData, source: Source): void { +export function add(node: Node, data: NodeInfo, source: Source): void { let id = getId(node, true); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); @@ -63,7 +63,7 @@ export function add(node: Node, data: INodeData, source: Source): void { track(id, source); } -export function update(node: Node, data: INodeData, source: Source): void { +export function update(node: Node, data: NodeInfo, source: Source): void { let id = getId(node); let parentId = node.parentElement ? getId(node.parentElement) : null; let nextId = getNextId(node); @@ -125,14 +125,14 @@ export function getNode(id: number): Node { return null; } -export function getValue(id: number): INodeValue { +export function getValue(id: number): NodeValue { if (id in values) { return values[id]; } return null; } -export function get(node: Node): INodeValue { +export function get(node: Node): NodeValue { let id = getId(node); return values[id]; } @@ -141,7 +141,7 @@ export function has(node: Node): boolean { return getId(node) in nodes; } -export function boxmodel(): INodeValue[] { +export function boxmodel(): NodeValue[] { let v = []; for (let id in values) { if (values[id].metadata.active && values[id].metadata.boxmodel) { @@ -151,7 +151,7 @@ export function boxmodel(): INodeValue[] { return v; } -export function updates(): INodeValue[] { +export function updates(): NodeValue[] { let output = []; for (let id of updateMap) { if (id in values) { @@ -166,7 +166,7 @@ export function updates(): INodeValue[] { return output; } -export function selectors(): INodeValue[] { +export function selectors(): NodeValue[] { let v = []; for (let id of selectorMap) { if (id in values) { @@ -186,7 +186,7 @@ function remove(id: number, source: Source): void { value.children = []; } -function selector(id: number, data: INodeData, parent: string): string { +function selector(id: number, data: NodeInfo, parent: string): string { switch (data.tag) { case "STYLE": case "TITLE": @@ -246,7 +246,7 @@ function getNextId(node: Node): number { return id; } -function copy(input: INodeValue[]): INodeValue[] { +function copy(input: NodeValue[]): NodeValue[] { return JSON.parse(JSON.stringify(input)); } @@ -268,7 +268,7 @@ function track(id: number, source: Source): void { } } -function history(id: number): INodeChange[] { +function history(id: number): NodeChange[] { if (id in changes) { return changes[id]; } diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 3fb18081..96c49522 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -1,15 +1,14 @@ -import {Event, Token} from "@clarity-types/data"; -import {INodeData} from "@clarity-types/layout"; -import {Metric} from "@clarity-types/metric"; +import {Event, Metric, Token} from "@clarity-types/data"; +import {NodeInfo} from "@clarity-types/layout"; import mask from "@src/core/mask"; import * as task from "@src/core/task"; import time from "@src/core/time"; import hash from "@src/data/hash"; +import * as metric from "@src/data/metric"; import {check} from "@src/data/token"; import { queue } from "@src/data/upload"; -import * as metric from "@src/metric"; import * as boxmodel from "./boxmodel"; -import {doc} from "./document"; +import * as doc from "./document"; import * as dom from "./dom"; export default async function(type: Event): Promise { @@ -17,7 +16,7 @@ export default async function(type: Event): Promise { let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; switch (type) { case Event.Document: - let d = doc; + let d = doc.data; tokens.push(d.width); tokens.push(d.height); metric.measure(Metric.DocumentWidth, d.width); @@ -32,15 +31,15 @@ export default async function(type: Event): Promise { } queue(tokens); break; - case Event.Checksum: + case Event.Hash: let selectors = dom.selectors(); let reference = 0; for (let value of selectors) { if (task.longtask(timer)) { await task.idle(timer); } - let checksum = hash(value.selector); - let pointer = tokens.indexOf(checksum); + let h = hash(value.selector); + let pointer = tokens.indexOf(h); tokens.push(value.id - reference); - tokens.push(pointer >= 0 ? [pointer] : checksum); + tokens.push(pointer >= 0 ? [pointer] : h); reference = value.id; } queue(tokens); @@ -51,7 +50,7 @@ export default async function(type: Event): Promise { for (let value of values) { if (task.longtask(timer)) { await task.idle(timer); } let metadata = []; - let data: INodeData = value.data; + let data: NodeInfo = value.data; let active = value.metadata.active; let keys = active ? ["tag", "path", "attributes", "value"] : ["tag"]; for (let key of keys) { diff --git a/src/layout/index.ts b/src/layout/index.ts index 59e064d5..4bf3cbe8 100644 --- a/src/layout/index.ts +++ b/src/layout/index.ts @@ -1,9 +1,11 @@ import * as boxmodel from "@src/layout/boxmodel"; import * as discover from "@src/layout/discover"; +import * as doc from "@src/layout/document"; import * as dom from "@src/layout/dom"; import * as mutation from "@src/layout/mutation"; export function start(): void { + doc.reset(); dom.reset(); mutation.start(); discover.start(); @@ -14,4 +16,5 @@ export function end(): void { dom.reset(); mutation.end(); boxmodel.reset(); + doc.reset(); } diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index 674268ac..a724d816 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -1,6 +1,5 @@ -import { Event } from "@clarity-types/data"; +import { Event, Metric } from "@clarity-types/data"; import { Source } from "@clarity-types/layout"; -import { Metric } from "@clarity-types/metric"; import config from "@src/core/config"; import * as task from "@src/core/task"; import * as boxmodel from "@src/layout/boxmodel"; @@ -105,7 +104,7 @@ async function process(): Promise { break; } } - await encode(config.lean ? Event.Checksum : Event.Mutation); + await encode(config.lean ? Event.Hash : Event.Mutation); task.stop(timer); } diff --git a/src/metric/encode.ts b/src/metric/encode.ts deleted file mode 100644 index 822ddbe3..00000000 --- a/src/metric/encode.ts +++ /dev/null @@ -1,25 +0,0 @@ -import {Token} from "@clarity-types/data"; -import { metrics, reset, updates } from "@src/metric"; - -export default function(last: boolean = false): Token[] { - let output = []; - - // Encode metrics - for (let metric in metrics) { - if (metrics[metric]) { - let m = num(metric); - if (updates.indexOf(m) >= 0 || last) { - output.push(m); - output.push(metrics[metric]); - } - } - } - - reset(); - - return output; -} - -function num(input: string): number { - return parseInt(input, 10); -} diff --git a/tslint.json b/tslint.json index c9420b70..ae7b0432 100644 --- a/tslint.json +++ b/tslint.json @@ -17,6 +17,7 @@ false ], "variable-name": false, + "interface-name": false, "interface-over-type-literal": false, "only-arrow-functions": false, "typedef": [ diff --git a/types/core.d.ts b/types/core.d.ts index e27d5201..a5e67ca2 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -1,16 +1,23 @@ -import { IClarityData, IPayload, Token } from "./data"; +import { ClarityInfo, Payload, Token } from "./data"; type TaskFunction = () => Promise; type TaskResolve = () => void; -export interface IEventBindingData { +/* Helper Interfaces */ + +export interface AsyncTask { + task: TaskFunction; + resolve: TaskResolve; +} + +export interface BrowserEvent { event: string; target: EventTarget; listener: EventListener; capture: boolean; } -export interface IConfig { +export interface Config { projectId?: string; longtask?: number; lookahead?: number; @@ -24,15 +31,10 @@ export interface IConfig { lean?: boolean; tokens?: string[]; url?: string; - onstart?: (data: IClarityData) => void; - upload?: (data: string, last: boolean) => void; + onstart?: (data: ClarityInfo) => void; + upload?: (data: string, sequence: number, last: boolean) => void; } -export interface ITaskTracker { +export interface TaskTiming { [key: number]: number; } - -export interface IAsyncTask { - task: TaskFunction; - resolve: TaskResolve; -} diff --git a/types/data.d.ts b/types/data.d.ts index a16a6ae1..e51f7dcd 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -1,17 +1,17 @@ -import { IMetric } from "./metric"; - export type Token = (string | number | number[] | string[]); export type DecodedToken = (any | any[]); +/* Enum */ + export const enum Event { - Page = 0, - Unload = 1, - Discover = 2, - Mutation = 3, - BoxModel = 4, - Checksum = 5, - Tag = 6, - Ping = 7, + Metric = 0, + Discover = 1, + Mutation = 2, + BoxModel = 3, + Hash = 4, + Resize = 5, + Document = 6, + Scroll = 7, Click = 8, MouseMove = 9, MouseDown = 10, @@ -24,18 +24,44 @@ export const enum Event { TouchMove = 17, TouchCancel = 18, Selection = 19, - Resize = 20, - Scroll = 21, - Change = 22, - Document = 23, - Visibility = 24, - Network = 25, - Performance = 26, - ScriptError = 27, - ImageError = 28, - Layout = 29, + Page = 20, + Tag = 21, + Ping = 22, + Unload = 23, + InputChange = 24, + Visibility = 25, + Network = 26, + Performance = 27, + ScriptError = 28, + ImageError = 29, Resource = 30, - Summary = 31 + Summary = 31, + Upload = 32 +} + +export const enum Metric { + Nodes = 0, + LayoutBytes = 1, + InteractionBytes = 2, + NetworkBytes = 3, + DiagnosticBytes = 4, + Mutations = 5, + Interactions = 6, + Clicks = 7, + Selections = 8, + Changes = 9, + ScriptErrors = 10, + ImageErrors = 11, + DiscoverTime = 12, + MutationTime = 13, + BoxModelTime = 14, + StartTime = 15, + ActiveTime = 16, + EndTime = 17, + ViewportWidth = 18, + ViewportHeight = 19, + DocumentWidth = 20, + DocumentHeight = 21 } export const enum Upload { @@ -44,65 +70,41 @@ export const enum Upload { Backup = 2 } -export const enum Flag { +export const enum BooleanFlag { False = 0, True = 1 } -export interface IPayload { +/* Helper Interfaces */ + +export interface Payload { e: Token[]; - m: Token[]; d: Token[][]; } -export interface IEncodedPayload { +export interface EncodedPayload { e: string; - m: string; d: string; } -export interface IDecodedPayload { - timestamp: number; - ua: string; - envelope: IEnvelope; - metrics: IMetric; - analytics: IDecodedEvent[]; - playback: IDecodedEvent[]; -} - -export interface IDecodedEvent { - time: number; - event: Event; - data: any; -} - -export interface ICookieData { +export interface CookieInfo { userId: string; sessionId: string; timestamp: number; } -export interface IClarityData { +export interface ClarityInfo { userId: string; sessionId: string; pageId: string; } -export interface IMetadata { - page: IPageData; - envelope: IEnvelope; -} - -export interface IPageData { - timestamp: number; - elapsed: number; - url: string; - title: string; - referrer: string; +export interface Metadata { + page: PageData; + envelope: Envelope; } -export interface IEnvelope { - elapsed: number; +export interface Envelope { sequence: number; version: string; projectId: string; @@ -110,24 +112,46 @@ export interface IEnvelope { sessionId: string; pageId: string; upload: Upload; - end: Flag; + end: BooleanFlag; } -export interface IPingData { +export interface Transit { + [key: number]: { + data: string; + attempts: number; + }; +} + +/* Event Data */ + +export interface MetricData { + [key: number]: number; +} + +export interface PageData { + timestamp: number; + ua: string; + url: string; + referrer: string; + lean: BooleanFlag; +} + +export interface PingData { gap: number; } -export interface ITagData { +export interface TagData { key: string; value: string; } -export interface IAugmentation { - timestamp: number; - ua: string; +export interface UploadData { + sequence: number; + attempts: number; + status: number; } -export interface IEventSummary { +export interface SummaryData { event: Event; start: number; end: number; diff --git a/types/decode/core.d.ts b/types/decode/core.d.ts new file mode 100644 index 00000000..5c8e37b5 --- /dev/null +++ b/types/decode/core.d.ts @@ -0,0 +1,6 @@ +import { Event } from "../data"; + +export interface PartialEvent { + time: number; + event: Event; +} diff --git a/types/decode/data.d.ts b/types/decode/data.d.ts new file mode 100644 index 00000000..9a48ecf3 --- /dev/null +++ b/types/decode/data.d.ts @@ -0,0 +1,13 @@ +import { MetricData, PageData, PingData, SummaryData, TagData, UploadData } from "../data"; +import { PartialEvent } from "./core"; + +/* Data Events */ +export interface MetricEvent extends PartialEvent { data: MetricData; } +export interface PageEvent extends PartialEvent { data: PageData; } +export interface PingEvent extends PartialEvent { data: PingData; } +export interface SummaryEvent extends PartialEvent { data: SummaryData[]; } +export interface TagEvent extends PartialEvent { data: TagData; } +export interface UploadEvent extends PartialEvent { data: UploadData; } +export interface DataEvent extends PartialEvent { + data: MetricData | PageData | PingData | SummaryData[] | TagData | UploadData; +} diff --git a/types/decode/decode.d.ts b/types/decode/decode.d.ts new file mode 100644 index 00000000..3d07e8c2 --- /dev/null +++ b/types/decode/decode.d.ts @@ -0,0 +1,34 @@ +import { Envelope, Event, MetricData, PageData, PingData, SummaryData, TagData, UploadData } from "../data"; +import { DataEvent, MetricEvent, PageEvent, PingEvent, SummaryEvent, TagEvent, UploadEvent } from "./data"; +import { BrokenImageEvent, DiagnosticEvent, ScriptErrorEvent } from "./diagnostic"; +import { InputChangeEvent, InteractionEvent, PointerEvent, ResizeEvent } from "./interaction"; +import { ScrollEvent, SelectionEvent, UnloadEvent, VisibilityEvent } from "./interaction"; +import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, LayoutEvent, ResourceEvent } from "./layout"; + +export type DecodedEvent = DataEvent | DiagnosticEvent | InteractionEvent | LayoutEvent; + +export interface DecodedPayload { + timestamp: number; + envelope: Envelope; + ua?: string; + metric?: MetricEvent[]; + page?: PageEvent[]; + ping?: PingEvent[]; + tag?: TagEvent[]; + image?: BrokenImageEvent[]; + script?: ScriptErrorEvent[]; + input?: InputChangeEvent[]; + pointer?: PointerEvent[]; + resize?: ResizeEvent[]; + scroll?: ScrollEvent[]; + selection?: SelectionEvent[]; + summary?: SummaryEvent[]; + unload?: UnloadEvent[]; + upload?: UploadEvent[]; + visibility?: VisibilityEvent[]; + boxmodel?: BoxModelEvent[]; + hash?: HashEvent[]; + resource?: ResourceEvent[]; + dom?: DomEvent[]; + doc?: DocumentEvent[]; +} diff --git a/types/decode/diagnostic.d.ts b/types/decode/diagnostic.d.ts new file mode 100644 index 00000000..9e6fd3c5 --- /dev/null +++ b/types/decode/diagnostic.d.ts @@ -0,0 +1,8 @@ +import { ImageErrorData, ScriptErrorData } from "../diagnostic"; +import { PartialEvent } from "./core"; + +export interface BrokenImageEvent extends PartialEvent { data: ImageErrorData; } +export interface ScriptErrorEvent extends PartialEvent { data: ScriptErrorData; } +export interface DiagnosticEvent extends PartialEvent { + data: ImageErrorData | ScriptErrorData; +} diff --git a/types/decode/index.d.ts b/types/decode/index.d.ts new file mode 100644 index 00000000..1d6f1b65 --- /dev/null +++ b/types/decode/index.d.ts @@ -0,0 +1,16 @@ +import { Payload } from "../data"; +import { DecodedPayload } from "./decode"; + +interface Decode { + decode: (data: string) => DecodedPayload; +} + +declare const decode: Decode; + +export * from "./data"; +export * from "./decode"; +export * from "./diagnostic"; +export * from "./layout"; +export * from "./interaction"; + +export { decode }; diff --git a/types/decode/interaction.d.ts b/types/decode/interaction.d.ts new file mode 100644 index 00000000..450f07ef --- /dev/null +++ b/types/decode/interaction.d.ts @@ -0,0 +1,13 @@ +import { InputChangeData, PointerData, ResizeData, ScrollData, SelectionData, UnloadData, VisibilityData } from "../interaction"; +import { PartialEvent } from "./core"; + +export interface InputChangeEvent extends PartialEvent { data: InputChangeData; } +export interface PointerEvent extends PartialEvent { data: PointerData; } +export interface ResizeEvent extends PartialEvent { data: ResizeData; } +export interface ScrollEvent extends PartialEvent { data: ScrollData; } +export interface SelectionEvent extends PartialEvent { data: SelectionData; } +export interface UnloadEvent extends PartialEvent { data: UnloadData; } +export interface VisibilityEvent extends PartialEvent { data: VisibilityData; } +export interface InteractionEvent extends PartialEvent { + data: InputChangeData | PointerData | ResizeData | ScrollData | SelectionData | UnloadData | VisibilityData; +} diff --git a/types/decode/layout.d.ts b/types/decode/layout.d.ts new file mode 100644 index 00000000..7ca1ab6e --- /dev/null +++ b/types/decode/layout.d.ts @@ -0,0 +1,21 @@ +import { Attributes, BoxModelData, DocumentData, HashData, ResourceData } from "../layout"; +import { PartialEvent } from "./core"; + +export interface BoxModelEvent extends PartialEvent { data: BoxModelData[]; } +export interface HashEvent extends PartialEvent { data: HashData[]; } +export interface DocumentEvent extends PartialEvent { data: DocumentData; } +export interface DomEvent extends PartialEvent { data: DomData[]; } +export interface ResourceEvent extends PartialEvent { data: ResourceData[]; } +export interface LayoutEvent extends PartialEvent { + data: BoxModelData[] | HashData[] | DocumentData | DomData[] | ResourceData[]; +} + +/* Event Data */ +export interface DomData { + id: number; + parent: number; + next: number; + tag: string; + attributes?: Attributes; + value?: string; +} diff --git a/types/diagnostic.d.ts b/types/diagnostic.d.ts index 27aa9b4d..e6dac9cd 100644 --- a/types/diagnostic.d.ts +++ b/types/diagnostic.d.ts @@ -1,4 +1,6 @@ -export interface IScriptError { +/* Event Data */ + +export interface ScriptErrorData { source: string; message: string; line: number; @@ -6,7 +8,7 @@ export interface IScriptError { stack: string; } -export interface IImageError { +export interface ImageErrorData { source: string; target: number; } diff --git a/types/index.d.ts b/types/index.d.ts index 5e9b5fd4..33105aa5 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -1,9 +1,9 @@ -import { IConfig } from "./core"; +import { Config } from "./core"; -interface IClarityJs { +interface Clarity { version: string; - config: (config?: IConfig) => boolean; - start: (config?: IConfig) => void; + config: (config?: Config) => boolean; + start: (config?: Config) => void; pause: () => void; resume: () => void; end: () => void; @@ -11,12 +11,12 @@ interface IClarityJs { tag: (key: string, value: string) => void; } -declare const clarity: IClarityJs; +declare const clarity: Clarity; export * from "./data"; export * from "./diagnostic"; export * from "./layout"; export * from "./interaction"; -export * from "./metric"; +export * from "./decode/index"; export { clarity }; diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 1fe13a57..3052e31d 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -1,4 +1,10 @@ -export interface IPointer { +/* Event Data */ +export interface InputChangeData { + target: number; + value: string; +} + +export interface PointerData { target: number; x: number; y: number; @@ -7,34 +13,29 @@ export interface IPointer { time?: number; } -export interface IResize { +export interface ResizeData { width: number; height: number; } -export interface IScroll { +export interface ScrollData { target: number; x: number; y: number; time?: number; } -export interface IVisibility { - visible: string; -} - -export interface ISelection { +export interface SelectionData { start: number; startOffset: number; end: number; endOffset: number; } -export interface IChange { - target: number; - value: string; +export interface UnloadData { + name: string; } -export interface IUnload { - name: string; +export interface VisibilityData { + visible: string; } diff --git a/types/layout.d.ts b/types/layout.d.ts index 88c259bb..022db9f2 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -1,3 +1,5 @@ +/* Enum */ + export const enum Source { Discover, ChildListAdd, @@ -6,70 +8,60 @@ export const enum Source { CharacterData } -export interface IAttributes { +/* Helper Interfaces */ + +export interface Attributes { [key: string]: string; } -export interface INodeData { +export interface NodeInfo { tag: string; path?: string; - attributes?: IAttributes; + attributes?: Attributes; value?: string; } -export interface INodeValue { +export interface NodeValue { id: number; parent: number; next: number; children: number[]; - data: INodeData; + data: NodeInfo; selector: string; - metadata: INodeMetadata; + metadata: NodeMeta; } -export interface INodeMetadata { +export interface NodeMeta { active: boolean; boxmodel: boolean; masked: boolean; } -export interface INodeChange { +export interface NodeChange { time: number; source: Source; - value: INodeValue; + value: NodeValue; } -export interface IDecodedNode { - id: number; - parent: number; - next: number; - tag: string; - attributes?: IAttributes; - value?: string; -} +/* Event Data */ -export interface IDocumentSize { +export interface DocumentData { width: number; height: number; } -export interface IBoxModel { +export interface BoxModelData { id: number; box: number[]; } -export interface IChecksum { - id: number; - checksum: string; -} - -export interface ILayout { +export interface HashData { id: number; - checksum: string; - selector: string; + hash: string; + selector?: string; } -export interface IResource { +export interface ResourceData { tag: string; url: string; } diff --git a/types/metric.d.ts b/types/metric.d.ts deleted file mode 100644 index 316ada16..00000000 --- a/types/metric.d.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Event } from "./data"; - -export const enum Metric { - Nodes = 0, - LayoutBytes = 1, - InteractionBytes = 2, - NetworkBytes = 3, - DiagnosticBytes = 4, - Mutations = 5, - Interactions = 6, - Clicks = 7, - Selections = 8, - Changes = 9, - ScriptErrors = 10, - ImageErrors = 11, - DiscoverTime = 12, - MutationTime = 13, - BoxModelTime = 14, - StartTime = 15, - ActiveTime = 16, - EndTime = 17, - ViewportWidth = 18, - ViewportHeight = 19, - DocumentWidth = 20, - DocumentHeight = 21 -} - -export interface IMetric { - [key: number]: number; -} From 964da6b74bcc9529fb1cace9cf506f7d653c88b6 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 4 Oct 2019 06:34:18 -0700 Subject: [PATCH 101/105] Resolving duplicate Ids using WeakMap (#150) * Fixing script errors * Script error fix * Using WeakMap instead of DOM property --- package.json | 2 +- src/clarity.ts | 8 +++++++- src/core/index.ts | 7 ++++++- src/core/version.ts | 2 +- src/data/index.ts | 1 + src/layout/boxmodel.ts | 6 ++++-- src/layout/dom.ts | 9 ++++++--- 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 84a0d3d6..3b0abf16 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.6", + "version": "1.0.0-beta.7", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/clarity.ts b/src/clarity.ts index 1cff76cb..6403826b 100644 --- a/src/clarity.ts +++ b/src/clarity.ts @@ -6,7 +6,6 @@ import * as data from "@src/data"; import * as diagnostic from "@src/diagnostic"; import * as interaction from "@src/interaction"; import * as layout from "@src/layout"; -export { tag } from "@src/data/tag"; let status = false; @@ -57,6 +56,13 @@ export function end(): void { } } +export function tag(key: string, value: string): void { + // Do not process tags if Clarity is not already activated + if (status) { + data.tag(key, value); + } +} + export function active(): boolean { return status; } diff --git a/src/core/index.ts b/src/core/index.ts index 896bd00b..39154b9c 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -14,7 +14,12 @@ export function end(): void { export function check(): boolean { try { - return typeof Promise !== "undefined" && MutationObserver && document["createTreeWalker"] && "now" in Date && "now" in performance; + return typeof Promise !== "undefined" && + MutationObserver && + document["createTreeWalker"] && + "now" in Date && + "now" in performance && + typeof WeakMap !== "undefined"; } catch (ex) { return false; } diff --git a/src/core/version.ts b/src/core/version.ts index 5e2d6bb9..40e66921 100644 --- a/src/core/version.ts +++ b/src/core/version.ts @@ -1,2 +1,2 @@ -let version = "1.0.0-b6"; +let version = "1.0.0-b7"; export default version; diff --git a/src/data/index.ts b/src/data/index.ts index 13dcf31f..54c331ce 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -3,6 +3,7 @@ import * as metric from "@src/data/metric"; import * as ping from "@src/data/ping"; import * as tag from "@src/data/tag"; import * as upload from "@src/data/upload"; +export { tag } from "@src/data/tag"; export function start(): void { upload.start(); diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index 5fa359f2..e32644f9 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -51,14 +51,16 @@ export function updates(): BoxModelData[] { export function relative(x: number, y: number, element: Element): number[] { if (x && x >= 0 && y && y >= 0 && element) { let box = layout(element); - return [x - box[0], y - box[1]]; + if (box !== null) { + return [x - box[0], y - box[1]]; + } } return [null, null]; } function update(id: number, box: number[]): void { let changed = box !== null; - if (id in bm) { + if (id in bm && box !== null && bm[id].box !== null) { changed = box.length === bm[id].box.length ? false : true; if (changed === false) { for (let i = 0; i < box.length; i++) { diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 33eeb49c..0bb93cc7 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -1,7 +1,6 @@ import { NodeChange, NodeInfo, NodeValue, Source } from "@clarity-types/layout"; import time from "@src/core/time"; -const NODE_ID_PROP: string = "__node_index__"; const DEVTOOLS_HOOK: string = "__CLARITY_DEVTOOLS_HOOK__"; const ID_ATTRIBUTE = "data-clarity"; const MASK_ATTRIBUTE = "data-clarity-mask"; @@ -13,6 +12,7 @@ let values: NodeValue[] = []; let changes: NodeChange[][] = []; let updateMap: number[] = []; let selectorMap: number[] = []; +let idMap: WeakMap = null; export function reset(): void { index = 1; @@ -21,15 +21,18 @@ export function reset(): void { updateMap = []; changes = []; selectorMap = []; + idMap = new WeakMap(); if (DEVTOOLS_HOOK in window) { window[DEVTOOLS_HOOK] = { get, getNode, history }; } } export function getId(node: Node, autogen: boolean = false): number { if (node === null) { return null; } - let id = node[NODE_ID_PROP]; + let id = idMap.get(node); if (!id && autogen) { - id = node[NODE_ID_PROP] = index++; + id = index++; + idMap.set(node, id); } + return id ? id : null; } From 8815a854e666fbd287f2e550e62233fae27985ca Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 13 Oct 2019 14:12:19 -0700 Subject: [PATCH 102/105] Bug fixes and data quality improvements over vBeta7 (#151) * Decoding selectors & bug fixes * Fixing script error and background mode * Updating hash code * Data quality bug fixes * Addressing PR feedback * Updating comments * Fixing upload retry logic * Bug fix in computing relative coordinates * Refactoring const strings into const enum * Refactoring upload hook * Changing type reference * Bug fixes and resolving script error when dom is still working at unload event --- decode/clarity.ts | 6 +-- decode/data.ts | 2 +- decode/layout.ts | 45 +++++++------------- decode/render.ts | 7 ++-- package.json | 2 +- src/core/mask.ts | 2 +- src/core/version.ts | 2 +- src/data/hash.ts | 22 ++++++---- src/data/metric.ts | 2 +- src/data/upload.ts | 80 ++++++++++++++++++++---------------- src/diagnostic/encode.ts | 29 +++++-------- src/diagnostic/image.ts | 15 +++---- src/diagnostic/index.ts | 3 +- src/diagnostic/script.ts | 10 ++--- src/interaction/pointer.ts | 20 ++++----- src/interaction/selection.ts | 29 +++++++++---- src/layout/boxmodel.ts | 11 +++-- src/layout/dom.ts | 51 ++++++++--------------- src/layout/node.ts | 4 +- src/layout/selector.ts | 23 +++++++++++ types/core.d.ts | 2 +- types/decode/decode.d.ts | 4 +- types/decode/diagnostic.d.ts | 2 +- types/layout.d.ts | 9 ++++ 24 files changed, 198 insertions(+), 184 deletions(-) create mode 100644 src/layout/selector.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 12b23124..ce453895 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -2,7 +2,7 @@ import version from "../src/core/version"; import { Event, Payload, Token } from "../types/data"; import { MetricEvent, PageEvent, PingEvent, SummaryEvent, TagEvent, UploadEvent } from "../types/decode/data"; import { DecodedEvent, DecodedPayload } from "../types/decode/decode"; -import { BrokenImageEvent, ScriptErrorEvent } from "../types/decode/diagnostic"; +import { ImageErrorEvent, ScriptErrorEvent } from "../types/decode/diagnostic"; import { InputChangeEvent, PointerEvent, ResizeEvent, ScrollEvent } from "../types/decode/interaction"; import { SelectionEvent, UnloadEvent, VisibilityEvent } from "../types/decode/interaction"; import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, ResourceEvent } from "../types/decode/layout"; @@ -114,7 +114,7 @@ export function decode(input: string): DecodedPayload { break; case Event.ImageError: if (payload.image === undefined) { payload.image = []; } - payload.image.push(diagnostic.decode(entry) as BrokenImageEvent); + payload.image.push(diagnostic.decode(entry) as ImageErrorEvent); break; default: console.error(`No handler for Event: ${JSON.stringify(entry)}`); @@ -124,7 +124,7 @@ export function decode(input: string): DecodedPayload { /* Enrich decoded payload with derived events */ payload.summary = data.summary() as SummaryEvent[]; - if (layout.hashes.length > 0) { payload.hash = layout.hash() as HashEvent[]; } + if (payload.dom && payload.dom.length > 0) { payload.hash = layout.hash() as HashEvent[]; } if (layout.resources.length > 0) { payload.resource = layout.resource() as ResourceEvent[]; } return payload; diff --git a/decode/data.ts b/decode/data.ts index 871dda8e..06943e4f 100644 --- a/decode/data.ts +++ b/decode/data.ts @@ -32,7 +32,7 @@ export function decode(tokens: Token[]): DataEvent { let upload: UploadData = { sequence: tokens[2] as number, attempts: tokens[3] as number, status: tokens[4] as number}; return { time, event, data: upload }; case Event.Metric: - let i = 0; + let i = 2; // Start from 3rd index since first two are used for time & event let metrics: MetricData = {}; while (i < tokens.length) { metrics[tokens[i++] as number] = tokens[i++] as number; diff --git a/decode/layout.ts b/decode/layout.ts index b42aafbe..720449ad 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -1,17 +1,17 @@ import generateHash from "../src/data/hash"; import { resolve } from "../src/data/token"; +import selector from "../src/layout/selector"; import { Event, Token } from "../types/data"; import { DomData, LayoutEvent } from "../types/decode/layout"; import { Attributes, BoxModelData, DocumentData, HashData, ResourceData } from "../types/layout"; -const ID_ATTRIBUTE = "data-clarity"; let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; -export let hashes: HashData[]; +export let hashes: { [key: number]: HashData } = {}; export let resources: ResourceData[]; let lastTime: number; export function reset(): void { - hashes = []; + hashes = {}; resources = []; lastTime = null; } @@ -90,7 +90,9 @@ export function decode(tokens: Token[]): LayoutEvent { } export function hash(): LayoutEvent[] { - return hashes.length > 0 ? [{ time: lastTime, event: Event.Hash, data: hashes }] : null; + let data = []; + for (let id in hashes) { if (hashes[id]) { data.push(hashes[id]); } } + return data.length > 0 ? [{ time: lastTime, event: Event.Hash, data }] : null; } export function resource(): LayoutEvent[] { @@ -105,9 +107,9 @@ function process(node: any[] | number[], tagIndex: number): DomData { tag: node[tagIndex] }; let hasAttribute = false; - let attributes = {}; + let attributes: Attributes = {}; let value = null; - let path = output.parent in hashes ? `${hashes[output.parent]}>` : null; + let prefix = output.parent in hashes ? `${hashes[output.parent].selector}>` : (output.parent ? "" : null); for (let i = tagIndex + 1; i < node.length; i++) { let token = node[i] as string; @@ -116,7 +118,7 @@ function process(node: any[] | number[], tagIndex: number): DomData { if (i === (node.length - 1) && output.tag === "STYLE") { value = token; } else if (lastChar === ">" && keyIndex === -1) { - path = token; + prefix = token; } else if (output.tag !== "*T" && keyIndex > 0) { hasAttribute = true; let k = token.substr(0, keyIndex); @@ -134,7 +136,9 @@ function process(node: any[] | number[], tagIndex: number): DomData { } } - getHash(output.id, output.tag, path, attributes); + let s = selector(output.tag, prefix, attributes); + if (s.length > 0) { hashes[output.id] = { id: output.id, hash: generateHash(s), selector: s }; } + getResource(output.tag, attributes); if (hasAttribute) { output.attributes = attributes; } if (value) { output.value = value; } @@ -142,25 +146,6 @@ function process(node: any[] | number[], tagIndex: number): DomData { return output; } -function getHash(id: number, tag: string, path: string, attributes: Attributes): void { - switch (tag) { - case "STYLE": - case "TITLE": - case "LINK": - case "META": - case "*T": - case "*D": - break; - default: - let s = path && path.length > 0 ? path + tag : tag; - if ("id" in attributes) { s = `${tag}#${attributes["id"]}`; } - if ("class" in attributes) { s += `.${attributes["class"].trim().split(" ").join(".")}`; } - if (ID_ATTRIBUTE in attributes) { s = `*${attributes[ID_ATTRIBUTE]}`; } - if (s) { hashes.push({ id, hash: generateHash(s), selector: s }); } - break; - } -} - function getResource(tag: string, attributes: Attributes): void { switch (tag) { case "LINK": @@ -174,9 +159,9 @@ function getResource(tag: string, attributes: Attributes): void { function unmask(value: string): string { let parts = value.split("*"); let placeholder = "x"; - if (parts.length === 2) { - let textCount = parseInt(parts[0], 36); - let wordCount = parseInt(parts[1], 36); + if (parts.length === 3 && parts[0] === "") { + let textCount = parseInt(parts[1], 36); + let wordCount = parseInt(parts[2], 36); if (isFinite(textCount) && isFinite(wordCount)) { if (wordCount > 0 && textCount === 0) { value = " "; diff --git a/decode/render.ts b/decode/render.ts index 4f28a7cd..c41b043b 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -1,12 +1,11 @@ import { Event, Metric, MetricData, PageData } from "../types/data"; import { DomData } from "../types/decode/layout"; import { InputChangeData, PointerData, ResizeData, ScrollData, SelectionData } from "../types/interaction"; -import { BoxModelData } from "../types/layout"; +import { BoxModelData, Constant } from "../types/layout"; let nodes = {}; let boxmodels = {}; let metrics: MetricData = null; -let svgns: string = "http://www.w3.org/2000/svg"; let lean = false; const METRIC_MAP = {}; METRIC_MAP[Metric.Nodes] = { name: "Node Count", unit: ""}; @@ -179,8 +178,8 @@ export function markup(data: DomData[], iframe: HTMLIFrameElement): void { } function createElement(doc: Document, tag: string, parent: HTMLElement): HTMLElement { - if (tag && tag.indexOf("svg:") === 0) { - return doc.createElementNS(svgns, tag) as HTMLElement; + if (tag && tag.indexOf(Constant.SVG_PREFIX) === 0) { + return doc.createElementNS(Constant.SVG_NAMESPACE as string, tag.substr(Constant.SVG_PREFIX.length)) as HTMLElement; } return doc.createElement(tag); } diff --git a/package.json b/package.json index 3b0abf16..5828f8a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.7", + "version": "1.0.0-beta.8", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/core/mask.ts b/src/core/mask.ts index ed0a1e2b..1934a9e0 100644 --- a/src/core/mask.ts +++ b/src/core/mask.ts @@ -10,5 +10,5 @@ export default function(value: string): string { wordCount += isWhiteSpace && !wasWhiteSpace ? 1 : 0; wasWhiteSpace = isWhiteSpace; } - return `${textCount.toString(36)}*${wordCount.toString(36)}`; + return `*${textCount.toString(36)}*${wordCount.toString(36)}`; } diff --git a/src/core/version.ts b/src/core/version.ts index 40e66921..5b047ec0 100644 --- a/src/core/version.ts +++ b/src/core/version.ts @@ -1,2 +1,2 @@ -let version = "1.0.0-b7"; +let version = "1.0.0-b8"; export default version; diff --git a/src/data/hash.ts b/src/data/hash.ts index 43af6bd5..11724708 100644 --- a/src/data/hash.ts +++ b/src/data/hash.ts @@ -1,11 +1,19 @@ // tslint:disable: no-bitwise export default function(input: string): string { - let value = 0; - for (let i = 0; i < input.length; i++) { - let char = input.charCodeAt(i); - value = ((value << 5) - value) + char; - value = value & value; // 32bit int conversion + // Code inspired from C# GetHashCode: https://github.com/Microsoft/referencesource/blob/master/mscorlib/system/string.cs + let hash = 0; + let hashOne = 5381; + let hashTwo = hashOne; + for (let i = 0; i < input.length; i += 2) { + let charOne = input.charCodeAt(i); + hashOne = ((hashOne << 5) + hashOne) ^ charOne; + if (i + 1 < input.length) { + let charTwo = input.charCodeAt(i + 1); + hashTwo = ((hashTwo << 5) + hashTwo) ^ charTwo; + } } - value = value & 0xfffffff; - return value.toString(36); + // Replace the magic number from C# implementation (1566083941) with a smaller prime number (11579) + // This ensures we don't hit integer overflow and prevent collisions + hash = Math.abs(hashOne + (hashTwo * 11579)); + return hash.toString(36).slice(-6); // Limit hashes to 6 characters } diff --git a/src/data/metric.ts b/src/data/metric.ts index a9d430b3..6e044eb1 100644 --- a/src/data/metric.ts +++ b/src/data/metric.ts @@ -9,7 +9,7 @@ export function start(): void { } export function end(): void { - data = null; + data = {}; } export function counter(metric: Metric, increment: number = 1): void { diff --git a/src/data/upload.ts b/src/data/upload.ts index 000b699e..8281b1f4 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -9,9 +9,11 @@ const MAX_RETRIES = 2; let events: string[]; let timeout: number = null; let transit: Transit; +let active: boolean; export let track: UploadData; export function start(): void { + active = true; events = []; transit = {}; track = null; @@ -19,36 +21,38 @@ export function start(): void { } export function queue(data: Token[]): void { - let type = data.length > 1 ? data[1] : null; - let event = JSON.stringify(data); - events.push(event); + if (active) { + let type = data.length > 1 ? data[1] : null; + let event = JSON.stringify(data); + events.push(event); - switch (type) { - case Event.Metric: - case Event.Upload: - return; // do not schedule upload callback - case Event.Discover: - case Event.Mutation: - case Event.BoxModel: - case Event.Hash: - case Event.Document: - metric.counter(Metric.LayoutBytes, event.length); - break; - case Event.Network: - case Event.Performance: - metric.counter(Metric.NetworkBytes, event.length); - break; - case Event.ScriptError: - case Event.ImageError: - metric.counter(Metric.DiagnosticBytes, event.length); - break; - default: - metric.counter(Metric.InteractionBytes, event.length); - break; - } + switch (type) { + case Event.Metric: + case Event.Upload: + return; // do not schedule upload callback + case Event.Discover: + case Event.Mutation: + case Event.BoxModel: + case Event.Hash: + case Event.Document: + metric.counter(Metric.LayoutBytes, event.length); + break; + case Event.Network: + case Event.Performance: + metric.counter(Metric.NetworkBytes, event.length); + break; + case Event.ScriptError: + case Event.ImageError: + metric.counter(Metric.DiagnosticBytes, event.length); + break; + default: + metric.counter(Metric.InteractionBytes, event.length); + break; + } - clearTimeout(timeout); - timeout = window.setTimeout(upload, config.delay); + clearTimeout(timeout); + timeout = window.setTimeout(upload, config.delay); + } } export function end(): void { @@ -57,16 +61,21 @@ export function end(): void { events = []; transit = {}; track = null; + active = false; } function upload(last: boolean = false): void { metric.compute(); - let handler = config.upload ? config.upload : send; let payload: EncodedPayload = {e: JSON.stringify(envelope(last)), d: `[${events.join()}]`}; + let data = stringify(payload); let sequence = metadata.envelope.sequence; - transit[sequence] = { data: stringify(payload), attempts: 1 }; - handler(transit[sequence].data, sequence, last); + send(data, sequence, last); if (last) { backup(payload); } else { ping.reset(); } + + // Send data to upload hook, if defined in the config + if (config.upload) { config.upload(data, sequence, last); } + + // Clear out events now that payload has been dispatched events = []; } @@ -75,10 +84,12 @@ function stringify(payload: EncodedPayload): string { } function send(data: string, sequence: number = null, last: boolean = false): void { + // Upload data if a valid URL is defined in the config if (config.url.length > 0) { if (last && "sendBeacon" in navigator) { navigator.sendBeacon(config.url, data); } else { + if (sequence in transit) { transit[sequence].attempts++; } else { transit[sequence] = { data, attempts: 1 }; } let xhr = new XMLHttpRequest(); xhr.open("POST", config.url); if (sequence !== null) { xhr.onreadystatechange = (): void => { check(xhr, sequence); }; } @@ -88,9 +99,8 @@ function send(data: string, sequence: number = null, last: boolean = false): voi } function check(xhr: XMLHttpRequest, sequence: number): void { - if (xhr.readyState === XMLHttpRequest.DONE) { + if (xhr && xhr.readyState === XMLHttpRequest.DONE && sequence in transit) { if ((xhr.status < 200 || xhr.status > 208) && transit[sequence].attempts <= MAX_RETRIES) { - transit[sequence].attempts++; send(transit[sequence].data, sequence); } else { track = { sequence, attempts: transit[sequence].attempts, status: xhr.status }; @@ -101,7 +111,7 @@ function check(xhr: XMLHttpRequest, sequence: number): void { } function recover(): void { - if (typeof localStorage !== "undefined") { + if (typeof localStorage !== "undefined" && config.url.length > 0) { let data = localStorage.getItem("clarity-backup"); if (data && data.length > 0) { send(data); @@ -110,7 +120,7 @@ function recover(): void { } function backup(payload: EncodedPayload): void { - if (typeof localStorage !== "undefined") { + if (typeof localStorage !== "undefined" && config.url.length > 0) { payload.e = JSON.stringify(envelope(true, true)); localStorage.setItem("clarity-backup", stringify(payload)); } diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts index a06d625d..65640b02 100644 --- a/src/diagnostic/encode.ts +++ b/src/diagnostic/encode.ts @@ -10,27 +10,18 @@ export default function(type: Event): Token[] { switch (type) { case Event.ScriptError: - let scripts = script.data; - for (let e of scripts) { - tokens.push(e.source); - tokens.push(e.message); - tokens.push(e.line); - tokens.push(e.column); - tokens.push(e.stack); - queue(tokens); - metric.counter(Metric.ScriptErrors); - } - script.reset(); + tokens.push(script.data.source); + tokens.push(script.data.message); + tokens.push(script.data.line); + tokens.push(script.data.column); + tokens.push(script.data.stack); + queue(tokens); + metric.counter(Metric.ScriptErrors); break; case Event.ImageError: - let images = image.data; - for (let e of images) { - tokens.push(e.source); - tokens.push(e.target); - queue(tokens); - metric.counter(Metric.ImageErrors); - } - image.reset(); + tokens.push(image.data.source); + tokens.push(image.data.target); + queue(tokens); break; } diff --git a/src/diagnostic/image.ts b/src/diagnostic/image.ts index 34324539..726874ad 100644 --- a/src/diagnostic/image.ts +++ b/src/diagnostic/image.ts @@ -4,7 +4,7 @@ import { bind } from "@src/core/event"; import { getId } from "@src/layout/dom"; import encode from "./encode"; -export let data: ImageErrorData[] = []; +export let data: ImageErrorData; export function start(): void { bind(document, "error", handler, true); @@ -13,15 +13,10 @@ export function start(): void { function handler(error: ErrorEvent): void { let target = error.target as HTMLElement; if (target && target.tagName === "IMG") { - data.push({ - source: error["filename"], + data = { + source: (target as HTMLImageElement).src, target: getId(target) - }); + }; + encode(Event.ImageError); } - - encode(Event.ImageError); -} - -export function reset(): void { - data = []; } diff --git a/src/diagnostic/index.ts b/src/diagnostic/index.ts index 760d583a..87aa5f67 100644 --- a/src/diagnostic/index.ts +++ b/src/diagnostic/index.ts @@ -7,6 +7,5 @@ export function start(): void { } export function end(): void { - script.reset(); - image.reset(); + /* cleanup operation */ } diff --git a/src/diagnostic/script.ts b/src/diagnostic/script.ts index 7754c552..83804c3a 100644 --- a/src/diagnostic/script.ts +++ b/src/diagnostic/script.ts @@ -3,7 +3,7 @@ import { ScriptErrorData } from "@clarity-types/diagnostic"; import { bind } from "@src/core/event"; import encode from "./encode"; -export let data: ScriptErrorData[] = []; +export let data: ScriptErrorData; export function start(): void { bind(window, "error", handler); @@ -12,17 +12,13 @@ export function start(): void { function handler(error: ErrorEvent): void { let e = error["error"] || error; - data.push({ + data = { message: e.message, stack: e.stack, line: error["lineno"], column: error["colno"], source: error["filename"] - }); + }; encode(Event.ScriptError); } - -export function reset(): void { - data = []; -} diff --git a/src/interaction/pointer.ts b/src/interaction/pointer.ts index e2c30208..3b8eb7f2 100644 --- a/src/interaction/pointer.ts +++ b/src/interaction/pointer.ts @@ -12,16 +12,16 @@ let timeout: number = null; export function start(): void { reset(); - bind(document, "mousedown", mouse.bind(this, Event.MouseDown)); - bind(document, "mouseup", mouse.bind(this, Event.MouseUp)); - bind(document, "mousemove", mouse.bind(this, Event.MouseMove)); - bind(document, "mousewheel", mouse.bind(this, Event.MouseWheel)); - bind(document, "dblclick", mouse.bind(this, Event.DoubleClick)); - bind(document, "click", mouse.bind(this, Event.Click)); - bind(document, "touchstart", touch.bind(this, Event.TouchStart)); - bind(document, "touchend", touch.bind(this, Event.TouchEnd)); - bind(document, "touchmove", touch.bind(this, Event.TouchMove)); - bind(document, "touchcancel", touch.bind(this, Event.TouchCancel)); + bind(document, "mousedown", mouse.bind(this, Event.MouseDown), true); + bind(document, "mouseup", mouse.bind(this, Event.MouseUp), true); + bind(document, "mousemove", mouse.bind(this, Event.MouseMove), true); + bind(document, "mousewheel", mouse.bind(this, Event.MouseWheel), true); + bind(document, "dblclick", mouse.bind(this, Event.DoubleClick), true); + bind(document, "click", mouse.bind(this, Event.Click), true); + bind(document, "touchstart", touch.bind(this, Event.TouchStart), true); + bind(document, "touchend", touch.bind(this, Event.TouchEnd), true); + bind(document, "touchmove", touch.bind(this, Event.TouchMove), true); + bind(document, "touchcancel", touch.bind(this, Event.TouchCancel), true); } function mouse(event: Event, evt: MouseEvent): void { diff --git a/src/interaction/selection.ts b/src/interaction/selection.ts index 5dadf2e2..007835be 100644 --- a/src/interaction/selection.ts +++ b/src/interaction/selection.ts @@ -6,7 +6,7 @@ import { getId } from "@src/layout/dom"; import encode from "./encode"; export let data: SelectionData = null; -let selection: Selection = null; +let previous: Selection = null; let timeout: number = null; export function start(): void { @@ -16,27 +16,38 @@ export function start(): void { } function recompute(): void { - let s = document.getSelection(); + let current = document.getSelection(); - if (selection !== null && data.start !== null && data.start !== getId(s.anchorNode)) { + // Bail out if we don't have a valid selection + if (current === null) { return; } + + let anchorNode = getId(current.anchorNode); + let focusNode = getId(current.focusNode); + + // Bail out if we got valid selection but not valid nodes + // In Edge, selectionchange gets fired even on interactions like right clicks and + // can result in null anchorNode and focusNode if there was no previous selection on page + if (anchorNode === null && focusNode === null) { return; } + + if (previous !== null && data.start !== null && data.start !== anchorNode) { clearTimeout(timeout); encode(Event.Selection); } data = { - start: getId(s.anchorNode), - startOffset: s.anchorOffset, - end: getId(s.focusNode), - endOffset: s.focusOffset + start: anchorNode, + startOffset: current.anchorOffset, + end: focusNode, + endOffset: current.focusOffset }; - selection = s; + previous = current; clearTimeout(timeout); timeout = window.setTimeout(encode, config.lookahead, Event.Selection); } export function reset(): void { - selection = null; + previous = null; data = { start: 0, startOffset: 0, end: 0, endOffset: 0 }; } diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index e32644f9..3d73ebdd 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -50,9 +50,14 @@ export function updates(): BoxModelData[] { export function relative(x: number, y: number, element: Element): number[] { if (x && x >= 0 && y && y >= 0 && element) { - let box = layout(element); - if (box !== null) { - return [x - box[0], y - box[1]]; + let scrollX = "pageXOffset" in window ? window.pageXOffset : document.documentElement.scrollLeft; + let scrollY = "pageYOffset" in window ? window.pageYOffset : document.documentElement.scrollTop; + let box = layout(element, scrollX, scrollY); + if (box) { + let [ex, ey, ew, eh]: number[] = box; + if (ew > 0 && eh > 0) { + return [((x - ex) / ew), ((y - ey) / eh)]; + } } } return [null, null]; diff --git a/src/layout/dom.ts b/src/layout/dom.ts index 0bb93cc7..b2a4a9f0 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -1,10 +1,7 @@ -import { NodeChange, NodeInfo, NodeValue, Source } from "@clarity-types/layout"; +import { Constant, NodeChange, NodeInfo, NodeValue, Source } from "@clarity-types/layout"; import time from "@src/core/time"; +import selector from "@src/layout/selector"; -const DEVTOOLS_HOOK: string = "__CLARITY_DEVTOOLS_HOOK__"; -const ID_ATTRIBUTE = "data-clarity"; -const MASK_ATTRIBUTE = "data-clarity-mask"; -const UNMASK_ATTRIBUTE = "data-clarity-unmask"; let index: number = 1; let nodes: Node[] = []; @@ -22,7 +19,7 @@ export function reset(): void { changes = []; selectorMap = []; idMap = new WeakMap(); - if (DEVTOOLS_HOOK in window) { window[DEVTOOLS_HOOK] = { get, getNode, history }; } + if (Constant.DEVTOOLS_HOOK in window) { window[Constant.DEVTOOLS_HOOK] = { get, getNode, history }; } } export function getId(node: Node, autogen: boolean = false): number { @@ -49,8 +46,8 @@ export function add(node: Node, data: NodeInfo, source: Source): void { masked = parent.metadata.masked; } - if (data.attributes && MASK_ATTRIBUTE in data.attributes) { masked = true; } - if (data.attributes && UNMASK_ATTRIBUTE in data.attributes) { masked = false; } + if (data.attributes && Constant.MASK_ATTRIBUTE in data.attributes) { masked = true; } + if (data.attributes && Constant.UNMASK_ATTRIBUTE in data.attributes) { masked = false; } nodes[id] = node; values[id] = { @@ -59,9 +56,10 @@ export function add(node: Node, data: NodeInfo, source: Source): void { next: nextId, children: [], data, - selector: selector(id, data, parent ? parent.selector : ""), + selector: "", metadata: { active: true, boxmodel: false, masked } }; + updateSelector(values[id]); layout(data.tag, id, parentId); track(id, source); } @@ -113,14 +111,21 @@ export function update(node: Node, data: NodeInfo, source: Source): void { } // Update selector - let parent = parentId && parentId in values ? values[parentId] : null; - value.selector = selector(id, data, parent ? parent.selector : ""), + updateSelector(value); layout(data.tag, id, parentId); track(id, source); } } +function updateSelector(value: NodeValue): void { + let prefix = value.parent && value.parent in values ? `${values[value.parent].selector}>` : null; + let ex = value.selector; + let current = selector(value.data.tag, prefix, value.data.attributes); + if (current !== ex && selectorMap.indexOf(value.id) === -1) { selectorMap.push(value.id); } + value.selector = current; +} + export function getNode(id: number): Node { if (id in nodes) { return nodes[id]; @@ -189,27 +194,6 @@ function remove(id: number, source: Source): void { value.children = []; } -function selector(id: number, data: NodeInfo, parent: string): string { - switch (data.tag) { - case "STYLE": - case "TITLE": - case "LINK": - case "META": - case "*T": - case "*D": - return ""; - default: - let value = getValue(id); - let ex = value ? value.selector : null; - let attributes = "attributes" in data ? data.attributes : {}; - let current = "id" in attributes ? `${data.tag}#${attributes.id}` : `${parent}>${data.tag}`; - if ("class" in attributes) { current = `${current}.${attributes.class.trim().split(" ").join(".")}`; } - if (ID_ATTRIBUTE in attributes) { current = `*${attributes[ID_ATTRIBUTE]}`; } - if (current !== ex && selectorMap.indexOf(id) === -1) { selectorMap.push(id); } - return current; - } -} - function layout(tag: string, id: number, parentId: number): void { if (id !== null && parentId !== null) { switch (tag) { @@ -229,7 +213,6 @@ function layout(tag: string, id: number, parentId: number): void { break; case "IMG": case "IFRAME": - case "svg:svg": values[id].metadata.boxmodel = true; break; default: @@ -263,7 +246,7 @@ function track(id: number, source: Source): void { updateMap.push(id); } else if (uIndex === -1) { updateMap.push(id); } - if (DEVTOOLS_HOOK in window) { + if (Constant.DEVTOOLS_HOOK in window) { let value = copy([values[id]])[0]; let change = { time: time(), source, value }; if (!(id in changes)) { changes[id] = []; } diff --git a/src/layout/node.ts b/src/layout/node.ts index d7bc9ca2..97952a44 100644 --- a/src/layout/node.ts +++ b/src/layout/node.ts @@ -1,4 +1,4 @@ -import { Source } from "@clarity-types/layout"; +import { Constant, Source } from "@clarity-types/layout"; import config from "@src/core/config"; import * as dom from "./dom"; @@ -39,7 +39,7 @@ export default function(node: Node, source: Source): void { case Node.ELEMENT_NODE: let element = (node as HTMLElement); let tag = element.tagName; - tag = (element.namespaceURI === "http://www.w3.org/2000/svg") ? "svg:" + tag : tag; + tag = (element.namespaceURI === Constant.SVG_NAMESPACE) ? Constant.SVG_PREFIX + tag : tag; switch (tag) { case "SCRIPT": case "NOSCRIPT": diff --git a/src/layout/selector.ts b/src/layout/selector.ts new file mode 100644 index 00000000..111823a4 --- /dev/null +++ b/src/layout/selector.ts @@ -0,0 +1,23 @@ +import { Attributes, Constant } from "../../types/layout"; + +export default function(tag: string, prefix: string, attributes: Attributes): string { + let empty = ""; + switch (tag) { + case "STYLE": + case "TITLE": + case "LINK": + case "META": + case "*T": + case "*D": + return empty; + case "HTML": + return "HTML"; + default: + if (prefix === null) { return empty; } + tag = tag.indexOf(Constant.SVG_PREFIX) === 0 ? tag.substr(Constant.SVG_PREFIX.length) : tag; + let s = "id" in attributes && attributes["id"].length > 0 ? `${tag}#${attributes.id}` : `${prefix}${tag}`; + if ("class" in attributes && attributes["class"].length > 0) { s = `${s}.${attributes.class.trim().split(/\s+/).join(".")}`; } + if (Constant.ID_ATTRIBUTE in attributes) { s = `*${attributes[Constant.ID_ATTRIBUTE]}`; } + return s; + } +} diff --git a/types/core.d.ts b/types/core.d.ts index a5e67ca2..1e9a6314 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -32,7 +32,7 @@ export interface Config { tokens?: string[]; url?: string; onstart?: (data: ClarityInfo) => void; - upload?: (data: string, sequence: number, last: boolean) => void; + upload?: (data: string, sequence?: number, last?: boolean) => void; } export interface TaskTiming { diff --git a/types/decode/decode.d.ts b/types/decode/decode.d.ts index 3d07e8c2..9650eaf6 100644 --- a/types/decode/decode.d.ts +++ b/types/decode/decode.d.ts @@ -1,6 +1,6 @@ import { Envelope, Event, MetricData, PageData, PingData, SummaryData, TagData, UploadData } from "../data"; import { DataEvent, MetricEvent, PageEvent, PingEvent, SummaryEvent, TagEvent, UploadEvent } from "./data"; -import { BrokenImageEvent, DiagnosticEvent, ScriptErrorEvent } from "./diagnostic"; +import { DiagnosticEvent, ImageErrorEvent, ScriptErrorEvent } from "./diagnostic"; import { InputChangeEvent, InteractionEvent, PointerEvent, ResizeEvent } from "./interaction"; import { ScrollEvent, SelectionEvent, UnloadEvent, VisibilityEvent } from "./interaction"; import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, LayoutEvent, ResourceEvent } from "./layout"; @@ -15,7 +15,7 @@ export interface DecodedPayload { page?: PageEvent[]; ping?: PingEvent[]; tag?: TagEvent[]; - image?: BrokenImageEvent[]; + image?: ImageErrorEvent[]; script?: ScriptErrorEvent[]; input?: InputChangeEvent[]; pointer?: PointerEvent[]; diff --git a/types/decode/diagnostic.d.ts b/types/decode/diagnostic.d.ts index 9e6fd3c5..d7d7df36 100644 --- a/types/decode/diagnostic.d.ts +++ b/types/decode/diagnostic.d.ts @@ -1,7 +1,7 @@ import { ImageErrorData, ScriptErrorData } from "../diagnostic"; import { PartialEvent } from "./core"; -export interface BrokenImageEvent extends PartialEvent { data: ImageErrorData; } +export interface ImageErrorEvent extends PartialEvent { data: ImageErrorData; } export interface ScriptErrorEvent extends PartialEvent { data: ScriptErrorData; } export interface DiagnosticEvent extends PartialEvent { data: ImageErrorData | ScriptErrorData; diff --git a/types/layout.d.ts b/types/layout.d.ts index 022db9f2..d4af0807 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -8,6 +8,15 @@ export const enum Source { CharacterData } +export const enum Constant { + SVG_PREFIX = "svg:", + SVG_NAMESPACE = "http://www.w3.org/2000/svg", + DEVTOOLS_HOOK = "__CLARITY_DEVTOOLS_HOOK__", + ID_ATTRIBUTE = "data-clarity", + MASK_ATTRIBUTE = "data-clarity-mask", + UNMASK_ATTRIBUTE = "data-clarity-unmask" +} + /* Helper Interfaces */ export interface Attributes { From 981616199dfabdfa93a57a0bf20d0f5ba6199e1b Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Fri, 18 Oct 2019 07:58:57 -0700 Subject: [PATCH 103/105] Incremental bug fixes over v1.0.0-beta8 (#153) * Removing local storage and fixing decode errors * Changing threshold for event summarization --- decode/clarity.ts | 2 +- decode/data.ts | 2 +- decode/layout.ts | 30 ++---------------------------- decode/render.ts | 35 ++++++++++++++++++++++++++++++++--- package.json | 2 +- src/core/index.ts | 2 +- src/core/version.ts | 2 +- src/data/metadata.ts | 6 +++--- src/data/upload.ts | 19 +------------------ src/layout/mutation.ts | 6 ++---- types/data.d.ts | 3 +-- 11 files changed, 46 insertions(+), 63 deletions(-) diff --git a/decode/clarity.ts b/decode/clarity.ts index ce453895..45beaba6 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -23,7 +23,7 @@ export function decode(input: string): DecodedPayload { let encoded: Token[][] = json.d; if (payload.envelope.version !== version) { - throw new Error(`Invalid Clarity Version. Actual: ${payload.envelope.version} | Expected: ${version}`); + throw new Error(`Invalid Clarity Version. Actual: ${payload.envelope.version} | Expected: ${version} | ${input.substr(0, 250)}`); } /* Reset components before decoding to keep them stateless */ diff --git a/decode/data.ts b/decode/data.ts index 06943e4f..f4320a5d 100644 --- a/decode/data.ts +++ b/decode/data.ts @@ -3,7 +3,7 @@ import { SummaryData, TagData, Token, Upload, UploadData } from "../types/data"; import { DataEvent } from "../types/decode/data"; let summaries: { [key: number]: SummaryData[] } = null; -const SUMMARY_THRESHOLD = 250; +const SUMMARY_THRESHOLD = 30; export function reset(): void { summaries = {}; diff --git a/decode/layout.ts b/decode/layout.ts index 720449ad..7f6f5d3e 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -122,7 +122,7 @@ function process(node: any[] | number[], tagIndex: number): DomData { } else if (output.tag !== "*T" && keyIndex > 0) { hasAttribute = true; let k = token.substr(0, keyIndex); - let v = unmask(token.substr(keyIndex + 1)); + let v = token.substr(keyIndex + 1); switch (k) { case "src": v = v.length === 0 ? placeholderImage : v; @@ -132,7 +132,7 @@ function process(node: any[] | number[], tagIndex: number): DomData { } attributes[k] = v; } else if (output.tag === "*T") { - value = unmask(token); + value = token; } } @@ -155,29 +155,3 @@ function getResource(tag: string, attributes: Attributes): void { break; } } - -function unmask(value: string): string { - let parts = value.split("*"); - let placeholder = "x"; - if (parts.length === 3 && parts[0] === "") { - let textCount = parseInt(parts[1], 36); - let wordCount = parseInt(parts[2], 36); - if (isFinite(textCount) && isFinite(wordCount)) { - if (wordCount > 0 && textCount === 0) { - value = " "; - } else if (wordCount === 0 && textCount > 0) { - value = Array(textCount + 1).join(placeholder); - } else if (wordCount > 0 && textCount > 0) { - value = ""; - let avg = Math.floor(textCount / wordCount); - while (value.length < textCount + wordCount) { - let gap = Math.min(avg, textCount + wordCount - value.length); - value += Array(gap + 1).join(placeholder) + " "; - } - } else { - value = Array(textCount + wordCount + 1).join(placeholder); - } - } - } - return value; -} diff --git a/decode/render.ts b/decode/render.ts index c41b043b..f8a77a50 100644 --- a/decode/render.ts +++ b/decode/render.ts @@ -129,7 +129,7 @@ export function markup(data: DomData[], iframe: HTMLIFrameElement): void { case "*T": let textElement = element(node.id); textElement = textElement ? textElement : doc.createTextNode(null); - textElement.nodeValue = node.value; + textElement.nodeValue = unmask(node.value); insert(node, parent, textElement, next); break; case "HTML": @@ -218,10 +218,11 @@ function setAttributes(node: HTMLElement, attributes: object): void { for (let attribute in attributes) { if (attributes[attribute] !== undefined) { try { + let v = unmask(attributes[attribute]); if (attribute.indexOf("xlink:") === 0) { - node.setAttributeNS("http://www.w3.org/1999/xlink", attribute, attributes[attribute]); + node.setAttributeNS("http://www.w3.org/1999/xlink", attribute, v); } else { - node.setAttribute(attribute, attributes[attribute]); + node.setAttribute(attribute, v); } } catch (ex) { console.warn("Node: " + node + " | " + JSON.stringify(attributes)); @@ -307,3 +308,31 @@ export function pointer(event: Event, data: PointerData, iframe: HTMLIFrameEleme function getNode(id: number): HTMLElement { return id in nodes ? nodes[id] : null; } + +function unmask(v: string): string { + if (v && v.length > 0) { + let parts = v.split("*"); + let placeholder = "x"; + if (parts.length === 3 && parts[0] === "") { + let textCount = parseInt(parts[1], 36); + let wordCount = parseInt(parts[2], 36); + if (isFinite(textCount) && isFinite(wordCount)) { + if (wordCount > 0 && textCount === 0) { + v = " "; + } else if (wordCount === 0 && textCount > 0) { + v = Array(textCount + 1).join(placeholder); + } else if (wordCount > 0 && textCount > 0) { + v = ""; + let avg = Math.floor(textCount / wordCount); + while (v.length < textCount + wordCount) { + let gap = Math.min(avg, textCount + wordCount - v.length); + v += Array(gap + 1).join(placeholder) + " "; + } + } else { + v = Array(textCount + wordCount + 1).join(placeholder); + } + } + } + } + return v; +} diff --git a/package.json b/package.json index 5828f8a5..76ee3b16 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.8", + "version": "1.0.0-beta.9", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/core/index.ts b/src/core/index.ts index 39154b9c..efe3d2b8 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -15,7 +15,7 @@ export function end(): void { export function check(): boolean { try { return typeof Promise !== "undefined" && - MutationObserver && + window["MutationObserver"] && document["createTreeWalker"] && "now" in Date && "now" in performance && diff --git a/src/core/version.ts b/src/core/version.ts index 5b047ec0..72b5bc4c 100644 --- a/src/core/version.ts +++ b/src/core/version.ts @@ -1,2 +1,2 @@ -let version = "1.0.0-b8"; +let version = "1.0.0-b9"; export default version; diff --git a/src/data/metadata.ts b/src/data/metadata.ts index 3a8545d1..3127f164 100644 --- a/src/data/metadata.ts +++ b/src/data/metadata.ts @@ -33,11 +33,11 @@ export function end(): void { metadata = null; } -export function envelope(last: boolean, backup: boolean = false): Token[] { +export function envelope(last: boolean): Token[] { let e = metadata.envelope; - e.upload = backup ? Upload.Backup : (last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async); + e.upload = last && "sendBeacon" in navigator ? Upload.Beacon : Upload.Async; e.end = last ? BooleanFlag.True : BooleanFlag.False; - if (e.upload !== Upload.Backup) { e.sequence++; } + e.sequence++; return [e.sequence, e.version, e.projectId, e.userId, e.sessionId, e.pageId, e.upload, e.end]; } diff --git a/src/data/upload.ts b/src/data/upload.ts index 8281b1f4..8c3a62fe 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -17,7 +17,6 @@ export function start(): void { events = []; transit = {}; track = null; - recover(); } export function queue(data: Token[]): void { @@ -70,7 +69,7 @@ function upload(last: boolean = false): void { let data = stringify(payload); let sequence = metadata.envelope.sequence; send(data, sequence, last); - if (last) { backup(payload); } else { ping.reset(); } + if (!last) { ping.reset(); } // Send data to upload hook, if defined in the config if (config.upload) { config.upload(data, sequence, last); } @@ -109,19 +108,3 @@ function check(xhr: XMLHttpRequest, sequence: number): void { } } } - -function recover(): void { - if (typeof localStorage !== "undefined" && config.url.length > 0) { - let data = localStorage.getItem("clarity-backup"); - if (data && data.length > 0) { - send(data); - } - } -} - -function backup(payload: EncodedPayload): void { - if (typeof localStorage !== "undefined" && config.url.length > 0) { - payload.e = JSON.stringify(envelope(true, true)); - localStorage.setItem("clarity-backup", stringify(payload)); - } -} diff --git a/src/layout/mutation.ts b/src/layout/mutation.ts index a724d816..0ec31e7e 100644 --- a/src/layout/mutation.ts +++ b/src/layout/mutation.ts @@ -13,9 +13,7 @@ let insertRule: (rule: string, index?: number) => number = null; let deleteRule: (index?: number) => void = null; export function start(): void { - if (observer) { - observer.disconnect(); - } + if (observer) { observer.disconnect(); } observer = window["MutationObserver"] ? new MutationObserver(handle) : null; observer.observe(document, { attributes: true, childList: true, characterData: true, subtree: true }); if (insertRule === null) { insertRule = CSSStyleSheet.prototype.insertRule; } @@ -38,7 +36,7 @@ export function start(): void { } export function end(): void { - observer.disconnect(); + if (observer) { observer.disconnect(); } observer = null; // Restoring original insertRule diff --git a/types/data.d.ts b/types/data.d.ts index e51f7dcd..804066c0 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -66,8 +66,7 @@ export const enum Metric { export const enum Upload { Async = 0, - Beacon = 1, - Backup = 2 + Beacon = 1 } export const enum BooleanFlag { From d590fd446d81dea0790c1241fd17ce775f02c674 Mon Sep 17 00:00:00 2001 From: Sarvesh Nagpal Date: Sun, 20 Oct 2019 19:50:48 -0700 Subject: [PATCH 104/105] Bug fixes and feature additions in v1.0.0-beta.10 (#154) * Enriching selectors with nth-of-type position * Bux fixes and changing position delimiter to "~" from "." * Adding fail safe check to automatically shutdown Clarity * Sending layout information for interaction targets * Simplifying Event.Target code logic * Refactoring logic to compute selector --- decode/clarity.ts | 6 ++++- decode/diagnostic.ts | 6 ++--- decode/interaction.ts | 4 ---- decode/layout.ts | 15 +++++++++--- package.json | 2 +- src/core/config.ts | 17 +++++++------- src/core/version.ts | 2 +- src/data/upload.ts | 13 +++++++++-- src/diagnostic/encode.ts | 2 +- src/diagnostic/script.ts | 2 +- src/interaction/change.ts | 10 ++++---- src/interaction/encode.ts | 4 ---- src/interaction/pointer.ts | 30 ++++++++++-------------- src/interaction/scroll.ts | 5 +++- src/interaction/selection.ts | 3 +++ src/layout/boxmodel.ts | 17 +------------- src/layout/dom.ts | 22 ++++++++++++++++-- src/layout/encode.ts | 12 +++++++++- src/layout/index.ts | 3 +++ src/layout/selector.ts | 16 +++++++++---- src/layout/target.ts | 44 ++++++++++++++++++++++++++++++++++++ types/core.d.ts | 1 + types/data.d.ts | 3 ++- types/decode/decode.d.ts | 3 ++- types/decode/layout.d.ts | 6 +++-- types/interaction.d.ts | 2 -- types/layout.d.ts | 7 ++++++ 27 files changed, 175 insertions(+), 82 deletions(-) create mode 100644 src/layout/target.ts diff --git a/decode/clarity.ts b/decode/clarity.ts index 45beaba6..29b1adcc 100644 --- a/decode/clarity.ts +++ b/decode/clarity.ts @@ -5,7 +5,7 @@ import { DecodedEvent, DecodedPayload } from "../types/decode/decode"; import { ImageErrorEvent, ScriptErrorEvent } from "../types/decode/diagnostic"; import { InputChangeEvent, PointerEvent, ResizeEvent, ScrollEvent } from "../types/decode/interaction"; import { SelectionEvent, UnloadEvent, VisibilityEvent } from "../types/decode/interaction"; -import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, ResourceEvent } from "../types/decode/layout"; +import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, ResourceEvent, TargetEvent } from "../types/decode/layout"; import * as data from "./data"; import * as diagnostic from "./diagnostic"; @@ -91,6 +91,10 @@ export function decode(input: string): DecodedPayload { if (payload.visibility === undefined) { payload.visibility = []; } payload.visibility.push(interaction.decode(entry) as VisibilityEvent); break; + case Event.Target: + if (payload.target === undefined) { payload.target = []; } + payload.target.push(layout.decode(entry) as TargetEvent); + break; case Event.BoxModel: if (payload.boxmodel === undefined) { payload.boxmodel = []; } payload.boxmodel.push(layout.decode(entry) as BoxModelEvent); diff --git a/decode/diagnostic.ts b/decode/diagnostic.ts index 40e05d95..c05343f9 100644 --- a/decode/diagnostic.ts +++ b/decode/diagnostic.ts @@ -12,13 +12,13 @@ export function decode(tokens: Token[]): DiagnosticEvent { target: tokens[3] as number }; return { time, event, data: imageError }; - case Event.Selection: + case Event.ScriptError: let scriptError: ScriptErrorData = { - source: tokens[2] as string, message: tokens[3] as string, line: tokens[4] as number, column: tokens[5] as number, - stack: tokens[6] as string + stack: tokens[6] as string, + source: tokens[2] as string }; return { time, event, data: scriptError }; } diff --git a/decode/interaction.ts b/decode/interaction.ts index f49ff96f..83db4678 100644 --- a/decode/interaction.ts +++ b/decode/interaction.ts @@ -18,10 +18,6 @@ export function decode(tokens: Token[]): InteractionEvent { case Event.TouchEnd: case Event.TouchMove: let pointerData: PointerData = { target: tokens[2] as number, x: tokens[3] as number, y: tokens[4] as number }; - if (tokens.length > 6) { - pointerData.targetX = tokens[5] as number; - pointerData.targetY = tokens[6] as number; - } return { time, event, data: pointerData }; case Event.Resize: let resizeData: ResizeData = { width: tokens[2] as number, height: tokens[3] as number }; diff --git a/decode/layout.ts b/decode/layout.ts index 7f6f5d3e..909d282b 100644 --- a/decode/layout.ts +++ b/decode/layout.ts @@ -3,7 +3,7 @@ import { resolve } from "../src/data/token"; import selector from "../src/layout/selector"; import { Event, Token } from "../types/data"; import { DomData, LayoutEvent } from "../types/decode/layout"; -import { Attributes, BoxModelData, DocumentData, HashData, ResourceData } from "../types/layout"; +import { Attributes, BoxModelData, DocumentData, HashData, ResourceData, TargetData } from "../types/layout"; let placeholderImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNiOAMAANUAz5n+TlUAAAAASUVORK5CYII="; export let hashes: { [key: number]: HashData } = {}; @@ -31,6 +31,13 @@ export function decode(tokens: Token[]): LayoutEvent { boxmodelData.push(boxmodel); } return { time, event, data: boxmodelData }; + case Event.Target: + let targetData: TargetData[] = []; + for (let i = 2; i < tokens.length; i += 3) { + let target: TargetData = { id: tokens[i] as number, hash: tokens[i + 1] as string, box: tokens[i + 2] as number[] }; + targetData.push(target); + } + return { time, event, data: targetData }; case Event.Hash: let reference = 0; let hashData: HashData[] = []; @@ -100,11 +107,13 @@ export function resource(): LayoutEvent[] { } function process(node: any[] | number[], tagIndex: number): DomData { + let [tag, position]: string[] = node[tagIndex] ? node[tagIndex].split("~") : [node[tagIndex]]; let output: DomData = { id: node[0], parent: tagIndex > 1 ? node[1] : null, next: tagIndex > 2 ? node[2] : null, - tag: node[tagIndex] + tag, + position: position ? parseInt(position, 10) : null }; let hasAttribute = false; let attributes: Attributes = {}; @@ -136,7 +145,7 @@ function process(node: any[] | number[], tagIndex: number): DomData { } } - let s = selector(output.tag, prefix, attributes); + let s = selector(output.tag, prefix, attributes, output.position); if (s.length > 0) { hashes[output.id] = { id: output.id, hash: generateHash(s), selector: s }; } getResource(output.tag, attributes); diff --git a/package.json b/package.json index 76ee3b16..92c8c73b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clarity-js", - "version": "1.0.0-beta.9", + "version": "1.0.0-beta.10", "description": "An analytics library that uses web page interactions to generate aggregated insights", "author": "Microsoft Corp.", "license": "MIT", diff --git a/src/core/config.ts b/src/core/config.ts index 8b673f47..6e974c87 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -2,14 +2,15 @@ import { Config } from "@clarity-types/core"; let config: Config = { projectId: null, - longtask: 30, - lookahead: 500, - distance: 20, - interval: 25, - delay: 1000, - expire: 7, - ping: 60 * 1000, - timeout: 10 * 60 * 1000, + longtask: 30, // 30 milliseconds + lookahead: 500, // 500 milliseconds + distance: 20, // 20 pixels + interval: 25, // 25 milliseconds + delay: 1000, // 1 second + expire: 7, // 7 days + ping: 60 * 1000, // 1 minute + timeout: 10 * 60 * 1000, // 10 minutes + shutdown: 2 * 60 * 60 * 1000, // 2 hours cssRules: false, lean: false, tokens: [], diff --git a/src/core/version.ts b/src/core/version.ts index 72b5bc4c..864cf815 100644 --- a/src/core/version.ts +++ b/src/core/version.ts @@ -1,2 +1,2 @@ -let version = "1.0.0-b9"; +let version = "1.0.0-b10"; export default version; diff --git a/src/data/upload.ts b/src/data/upload.ts index 8c3a62fe..e9346ff9 100644 --- a/src/data/upload.ts +++ b/src/data/upload.ts @@ -1,5 +1,6 @@ import { EncodedPayload, Event, Metric, Token, Transit, UploadData } from "@clarity-types/data"; import config from "@src/core/config"; +import time from "@src/core/time"; import encode from "@src/data/encode"; import { envelope, metadata } from "@src/data/metadata"; import * as metric from "@src/data/metric"; @@ -34,6 +35,7 @@ export function queue(data: Token[]): void { case Event.BoxModel: case Event.Hash: case Event.Document: + case Event.Target: metric.counter(Metric.LayoutBytes, event.length); break; case Event.Network: @@ -49,8 +51,15 @@ export function queue(data: Token[]): void { break; } - clearTimeout(timeout); - timeout = window.setTimeout(upload, config.delay); + // This is a precautionary check acting as a fail safe mechanism to get out of + // unexpected situations. Ideally, expectation is that pause / resume will work as designed. + // However, in some cases involving script errors, we may fail to pause Clarity instrumentation. + // In those edge cases, we will cut the cord after a configurable shutdown value. + // The only exception is the very last payload, for which we will attempt one final delivery to the server. + if (time() < config.shutdown) { + clearTimeout(timeout); + timeout = window.setTimeout(upload, config.delay); + } } } diff --git a/src/diagnostic/encode.ts b/src/diagnostic/encode.ts index 65640b02..2f614d95 100644 --- a/src/diagnostic/encode.ts +++ b/src/diagnostic/encode.ts @@ -10,11 +10,11 @@ export default function(type: Event): Token[] { switch (type) { case Event.ScriptError: - tokens.push(script.data.source); tokens.push(script.data.message); tokens.push(script.data.line); tokens.push(script.data.column); tokens.push(script.data.stack); + tokens.push(script.data.source); queue(tokens); metric.counter(Metric.ScriptErrors); break; diff --git a/src/diagnostic/script.ts b/src/diagnostic/script.ts index 83804c3a..03aef14a 100644 --- a/src/diagnostic/script.ts +++ b/src/diagnostic/script.ts @@ -14,9 +14,9 @@ function handler(error: ErrorEvent): void { data = { message: e.message, - stack: e.stack, line: error["lineno"], column: error["colno"], + stack: e.stack, source: error["filename"] }; diff --git a/src/interaction/change.ts b/src/interaction/change.ts index 0e0d5fca..b623acaa 100644 --- a/src/interaction/change.ts +++ b/src/interaction/change.ts @@ -3,6 +3,7 @@ import { InputChangeData } from "@clarity-types/interaction"; import { bind } from "@src/core/event"; import mask from "@src/core/mask"; import { get } from "@src/layout/dom"; +import * as target from "@src/layout/target"; import encode from "./encode"; export let data: InputChangeData; @@ -12,10 +13,11 @@ export function start(): void { } function recompute(evt: UIEvent): void { - let target = evt.target as HTMLInputElement; - let value = get(target); - if (target && value) { - data = { target: value.id, value: value.metadata.masked ? mask(target.value) : target.value }; + let input = evt.target as HTMLInputElement; + let value = get(input); + if (input && value) { + target.observe(value.id); + data = { target: value.id, value: value.metadata.masked ? mask(input.value) : input.value }; encode(Event.InputChange); } } diff --git a/src/interaction/encode.ts b/src/interaction/encode.ts index 800c9b53..6d0f5c65 100644 --- a/src/interaction/encode.ts +++ b/src/interaction/encode.ts @@ -31,10 +31,6 @@ export default function(type: Event): void { tokens.push(entry.target); tokens.push(entry.x); tokens.push(entry.y); - if (entry.targetX && entry.targetY) { - tokens.push(entry.targetX); - tokens.push(entry.targetY); - } queue(tokens); } pointer.reset(); diff --git a/src/interaction/pointer.ts b/src/interaction/pointer.ts index 3b8eb7f2..331cef54 100644 --- a/src/interaction/pointer.ts +++ b/src/interaction/pointer.ts @@ -3,8 +3,8 @@ import { PointerData } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; -import * as boxmodel from "@src/layout/boxmodel"; import { getId } from "@src/layout/dom"; +import * as target from "@src/layout/target"; import encode from "./encode"; export let data: { [key: number]: PointerData[] } = {}; @@ -28,30 +28,24 @@ function mouse(event: Event, evt: MouseEvent): void { let de = document.documentElement; let x = "pageX" in evt ? Math.round(evt.pageX) : ("clientX" in evt ? Math.round(evt["clientX"] + de.scrollLeft) : null); let y = "pageY" in evt ? Math.round(evt.pageY) : ("clientY" in evt ? Math.round(evt["clientY"] + de.scrollTop) : null); - let target = evt.target ? getId(evt.target as Node) : null; - let targetX = null; // x coordinate relative to the target element - let targetY = null; // y coordinate relative to the target element - if (event === Event.Click) { - // Populate (x,y) relative to target only when the above condition is met - // It's an expensive operation and we can expand the scope as desired later - event = evt.buttons === 2 || evt.button === 2 ? Event.RightClick : event; - let relative = boxmodel.relative(x, y, evt.target as Element); - targetX = relative[0]; - targetY = relative[1]; - } - handler(event, {target, x, y, targetX, targetY, time: time()}); + let id = evt.target ? getId(evt.target as Node) : null; + target.observe(id); + event = event === Event.Click && (evt.buttons === 2 || evt.button === 2) ? Event.RightClick : event; + handler(event, {target: id, x, y, time: time()}); } function touch(event: Event, evt: TouchEvent): void { let de = document.documentElement; let touches = evt.changedTouches; - let target = evt.target ? getId(evt.target as Node) : null; + let id = evt.target ? getId(evt.target as Node) : null; + let t = time(); + target.observe(id); if (touches) { for (let i = 0; i < touches.length; i++) { - let t = touches[i]; - let x = "clientX" in t ? Math.round(t["clientX"] + de.scrollLeft) : null; - let y = "clientY" in t ? Math.round(t["clientY"] + de.scrollTop) : null; - handler(event, {target, x, y, time: time()}); + let entry = touches[i]; + let x = "clientX" in entry ? Math.round(entry["clientX"] + de.scrollLeft) : null; + let y = "clientY" in entry ? Math.round(entry["clientY"] + de.scrollTop) : null; + handler(event, {target: id, x, y, time: t}); } } } diff --git a/src/interaction/scroll.ts b/src/interaction/scroll.ts index edd522bd..c9a388a3 100644 --- a/src/interaction/scroll.ts +++ b/src/interaction/scroll.ts @@ -4,6 +4,7 @@ import config from "@src/core/config"; import { bind } from "@src/core/event"; import time from "@src/core/time"; import { getId } from "@src/layout/dom"; +import * as target from "@src/layout/target"; import encode from "./encode"; export let data: ScrollData[] = []; @@ -18,7 +19,9 @@ function recompute(event: UIEvent = null): void { let eventTarget = event ? (event.target === document ? document.documentElement : event.target) : document.documentElement; let x = Math.round((eventTarget as HTMLElement).scrollLeft); let y = Math.round((eventTarget as HTMLElement).scrollTop); - let current: ScrollData = {target: getId(eventTarget as Node), x, y, time: time()}; + let id = getId(eventTarget as Node); + target.observe(id); + let current: ScrollData = {target: id, x, y, time: time()}; let length = data.length; let last = length > 1 ? data[length - 2] : null; diff --git a/src/interaction/selection.ts b/src/interaction/selection.ts index 007835be..986f2fe7 100644 --- a/src/interaction/selection.ts +++ b/src/interaction/selection.ts @@ -3,6 +3,7 @@ import { SelectionData } from "@clarity-types/interaction"; import config from "@src/core/config"; import { bind } from "@src/core/event"; import { getId } from "@src/layout/dom"; +import * as target from "@src/layout/target"; import encode from "./encode"; export let data: SelectionData = null; @@ -31,6 +32,8 @@ function recompute(): void { if (previous !== null && data.start !== null && data.start !== anchorNode) { clearTimeout(timeout); + target.observe(data.start); + target.observe(data.end); encode(Event.Selection); } diff --git a/src/layout/boxmodel.ts b/src/layout/boxmodel.ts index 3d73ebdd..05a74330 100644 --- a/src/layout/boxmodel.ts +++ b/src/layout/boxmodel.ts @@ -48,21 +48,6 @@ export function updates(): BoxModelData[] { return summary; } -export function relative(x: number, y: number, element: Element): number[] { - if (x && x >= 0 && y && y >= 0 && element) { - let scrollX = "pageXOffset" in window ? window.pageXOffset : document.documentElement.scrollLeft; - let scrollY = "pageYOffset" in window ? window.pageYOffset : document.documentElement.scrollTop; - let box = layout(element, scrollX, scrollY); - if (box) { - let [ex, ey, ew, eh]: number[] = box; - if (ew > 0 && eh > 0) { - return [((x - ex) / ew), ((y - ey) / eh)]; - } - } - } - return [null, null]; -} - function update(id: number, box: number[]): void { let changed = box !== null; if (id in bm && box !== null && bm[id].box !== null) { @@ -83,7 +68,7 @@ function update(id: number, box: number[]): void { } } -function layout(element: Element, x: number = 0, y: number = 0): number[] { +export function layout(element: Element, x: number = 0, y: number = 0): number[] { let box: number[] = null; let rect = element.getBoundingClientRect(); diff --git a/src/layout/dom.ts b/src/layout/dom.ts index b2a4a9f0..96720dcb 100644 --- a/src/layout/dom.ts +++ b/src/layout/dom.ts @@ -55,6 +55,7 @@ export function add(node: Node, data: NodeInfo, source: Source): void { parent: parentId, next: nextId, children: [], + position: null, data, selector: "", metadata: { active: true, boxmodel: false, masked } @@ -118,10 +119,27 @@ export function update(node: Node, data: NodeInfo, source: Source): void { } } +function position(parent: NodeValue, child: NodeValue): number { + let tag = child.data.tag; + // Find relative position of the element to generate :nth-of-type selector + // We restrict relative positioning to handful of tags for now. + if (parent && (tag === "DIV" || tag === "TR" || tag === "P" || tag === "LI")) { + child.position = 1; + let idx = parent ? parent.children.indexOf(child.id) : -1; + while (idx-- > 0) { + let sibling = values[parent.children[idx]]; + if (child.data.tag === sibling.data.tag) { child.position = sibling.position + 1; } + break; + } + } + return child.position; +} + function updateSelector(value: NodeValue): void { - let prefix = value.parent && value.parent in values ? `${values[value.parent].selector}>` : null; + let parent = value.parent && value.parent in values ? values[value.parent] : null; + let prefix = parent ? `${parent.selector}>` : null; let ex = value.selector; - let current = selector(value.data.tag, prefix, value.data.attributes); + let current = selector(value.data.tag, prefix, value.data.attributes, position(parent, value)); if (current !== ex && selectorMap.indexOf(value.id) === -1) { selectorMap.push(value.id); } value.selector = current; } diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 96c49522..141ccb9b 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -10,6 +10,7 @@ import { queue } from "@src/data/upload"; import * as boxmodel from "./boxmodel"; import * as doc from "./document"; import * as dom from "./dom"; +import * as target from "./target"; export default async function(type: Event): Promise { let tokens: Token[] = [time(), type]; @@ -31,6 +32,15 @@ export default async function(type: Event): Promise { } queue(tokens); break; + case Event.Target: + let targets = target.updates(); + for (let value of targets) { + tokens.push(value.id); + tokens.push(value.hash); + tokens.push(value.box); + } + queue(tokens); + break; case Event.Hash: let selectors = dom.selectors(); let reference = 0; @@ -61,7 +71,7 @@ export default async function(type: Event): Promise { tokens.push(value.id); if (value.parent && active) { tokens.push(value.parent); } if (value.next && active) { tokens.push(value.next); } - metadata.push(data[key]); + metadata.push(value.position ? `${data[key]}~${value.position}` : data[key]); break; case "path": metadata.push(`${value.data.path}>`); diff --git a/src/layout/index.ts b/src/layout/index.ts index 4bf3cbe8..bc170081 100644 --- a/src/layout/index.ts +++ b/src/layout/index.ts @@ -3,6 +3,7 @@ import * as discover from "@src/layout/discover"; import * as doc from "@src/layout/document"; import * as dom from "@src/layout/dom"; import * as mutation from "@src/layout/mutation"; +import * as target from "@src/layout/target"; export function start(): void { doc.reset(); @@ -10,11 +11,13 @@ export function start(): void { mutation.start(); discover.start(); boxmodel.reset(); + target.reset(); } export function end(): void { dom.reset(); mutation.end(); boxmodel.reset(); + target.reset(); doc.reset(); } diff --git a/src/layout/selector.ts b/src/layout/selector.ts index 111823a4..f9da4519 100644 --- a/src/layout/selector.ts +++ b/src/layout/selector.ts @@ -1,7 +1,8 @@ import { Attributes, Constant } from "../../types/layout"; -export default function(tag: string, prefix: string, attributes: Attributes): string { +export default function(tag: string, prefix: string, attributes: Attributes, position: number): string { let empty = ""; + let suffix = position ? `:nth-of-type(${position})` : empty; switch (tag) { case "STYLE": case "TITLE": @@ -15,9 +16,14 @@ export default function(tag: string, prefix: string, attributes: Attributes): st default: if (prefix === null) { return empty; } tag = tag.indexOf(Constant.SVG_PREFIX) === 0 ? tag.substr(Constant.SVG_PREFIX.length) : tag; - let s = "id" in attributes && attributes["id"].length > 0 ? `${tag}#${attributes.id}` : `${prefix}${tag}`; - if ("class" in attributes && attributes["class"].length > 0) { s = `${s}.${attributes.class.trim().split(/\s+/).join(".")}`; } - if (Constant.ID_ATTRIBUTE in attributes) { s = `*${attributes[Constant.ID_ATTRIBUTE]}`; } - return s; + let selector = `${prefix}${tag}${suffix}`; + if (Constant.ID_ATTRIBUTE in attributes) { + selector = `*${attributes[Constant.ID_ATTRIBUTE]}`; + } else if ("id" in attributes && attributes["id"].length > 0) { + selector = `${tag}#${attributes.id}`; + } else if ("class" in attributes && attributes["class"].length > 0) { + selector = `${prefix}${tag}.${attributes.class.trim().split(/\s+/).join(".")}${suffix}`; + } + return selector; } } diff --git a/src/layout/target.ts b/src/layout/target.ts new file mode 100644 index 00000000..be04b593 --- /dev/null +++ b/src/layout/target.ts @@ -0,0 +1,44 @@ +import { Event } from "@clarity-types/data"; +import { TargetData } from "@clarity-types/layout"; +import config from "@src/core/config"; +import hash from "@src/data/hash"; +import { layout } from "@src/layout/boxmodel"; +import encode from "@src/layout/encode"; +import * as dom from "./dom"; + +let queue: number[] = []; +let timeout: number = null; + +export function reset(): void { + queue = []; + clearTimeout(timeout); + timeout = null; +} + +export function observe(id: number): void { + if (queue.indexOf(id) === -1) { queue.push(id); } + clearTimeout(timeout); + timeout = window.setTimeout(encode, config.lookahead, Event.Target); +} + +export function updates(): TargetData[] { + let data: TargetData[] = []; + if (queue.length > 0) { + let doc = document.documentElement; + let x = "pageXOffset" in window ? window.pageXOffset : doc.scrollLeft; + let y = "pageYOffset" in window ? window.pageYOffset : doc.scrollTop; + + // Process all layout computations in single batch to avoid reflows + for (let id of queue) { + let value = dom.getValue(id); + let node = dom.getNode(id) as Element; + data.push({ + id, + hash: value ? hash(value.selector) : "", + box: node && node.nodeType !== Node.TEXT_NODE ? layout(node, x, y) : [] + }); + } + reset(); + } + return data; +} diff --git a/types/core.d.ts b/types/core.d.ts index 1e9a6314..0c6779dd 100644 --- a/types/core.d.ts +++ b/types/core.d.ts @@ -27,6 +27,7 @@ export interface Config { expire?: number; ping?: number; timeout?: number; + shutdown?: number; cssRules?: boolean; lean?: boolean; tokens?: string[]; diff --git a/types/data.d.ts b/types/data.d.ts index 804066c0..da97b9c4 100644 --- a/types/data.d.ts +++ b/types/data.d.ts @@ -36,7 +36,8 @@ export const enum Event { ImageError = 29, Resource = 30, Summary = 31, - Upload = 32 + Upload = 32, + Target = 33 } export const enum Metric { diff --git a/types/decode/decode.d.ts b/types/decode/decode.d.ts index 9650eaf6..8fe35b5e 100644 --- a/types/decode/decode.d.ts +++ b/types/decode/decode.d.ts @@ -3,7 +3,7 @@ import { DataEvent, MetricEvent, PageEvent, PingEvent, SummaryEvent, TagEvent, U import { DiagnosticEvent, ImageErrorEvent, ScriptErrorEvent } from "./diagnostic"; import { InputChangeEvent, InteractionEvent, PointerEvent, ResizeEvent } from "./interaction"; import { ScrollEvent, SelectionEvent, UnloadEvent, VisibilityEvent } from "./interaction"; -import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, LayoutEvent, ResourceEvent } from "./layout"; +import { BoxModelEvent, DocumentEvent, DomEvent, HashEvent, LayoutEvent, ResourceEvent, TargetEvent } from "./layout"; export type DecodedEvent = DataEvent | DiagnosticEvent | InteractionEvent | LayoutEvent; @@ -31,4 +31,5 @@ export interface DecodedPayload { resource?: ResourceEvent[]; dom?: DomEvent[]; doc?: DocumentEvent[]; + target?: TargetEvent[]; } diff --git a/types/decode/layout.d.ts b/types/decode/layout.d.ts index 7ca1ab6e..c58363fb 100644 --- a/types/decode/layout.d.ts +++ b/types/decode/layout.d.ts @@ -1,4 +1,4 @@ -import { Attributes, BoxModelData, DocumentData, HashData, ResourceData } from "../layout"; +import { Attributes, BoxModelData, DocumentData, HashData, ResourceData, TargetData } from "../layout"; import { PartialEvent } from "./core"; export interface BoxModelEvent extends PartialEvent { data: BoxModelData[]; } @@ -6,8 +6,9 @@ export interface HashEvent extends PartialEvent { data: HashData[]; } export interface DocumentEvent extends PartialEvent { data: DocumentData; } export interface DomEvent extends PartialEvent { data: DomData[]; } export interface ResourceEvent extends PartialEvent { data: ResourceData[]; } +export interface TargetEvent extends PartialEvent { data: TargetData[]; } export interface LayoutEvent extends PartialEvent { - data: BoxModelData[] | HashData[] | DocumentData | DomData[] | ResourceData[]; + data: BoxModelData[] | HashData[] | DocumentData | DomData[] | ResourceData[] | TargetData[]; } /* Event Data */ @@ -16,6 +17,7 @@ export interface DomData { parent: number; next: number; tag: string; + position: number; attributes?: Attributes; value?: string; } diff --git a/types/interaction.d.ts b/types/interaction.d.ts index 3052e31d..ffa88f8c 100644 --- a/types/interaction.d.ts +++ b/types/interaction.d.ts @@ -8,8 +8,6 @@ export interface PointerData { target: number; x: number; y: number; - targetX?: number; - targetY?: number; time?: number; } diff --git a/types/layout.d.ts b/types/layout.d.ts index d4af0807..ab083eb2 100644 --- a/types/layout.d.ts +++ b/types/layout.d.ts @@ -34,6 +34,7 @@ export interface NodeValue { id: number; parent: number; next: number; + position: number; children: number[]; data: NodeInfo; selector: string; @@ -64,6 +65,12 @@ export interface BoxModelData { box: number[]; } +export interface TargetData { + id: number; + hash: string; + box: number[]; +} + export interface HashData { id: number; hash: string; From 80fd07132d40053a1f8531980eeb983bb7363497 Mon Sep 17 00:00:00 2001 From: saiphcita Date: Wed, 30 Oct 2019 13:35:10 -0700 Subject: [PATCH 105/105] adds flags to disable clarity tracking in certain interface elements --- src/layout/encode.ts | 284 ++++++++++++++++++++++--------------------- 1 file changed, 147 insertions(+), 137 deletions(-) diff --git a/src/layout/encode.ts b/src/layout/encode.ts index 141ccb9b..2537716d 100644 --- a/src/layout/encode.ts +++ b/src/layout/encode.ts @@ -1,137 +1,147 @@ -import {Event, Metric, Token} from "@clarity-types/data"; -import {NodeInfo} from "@clarity-types/layout"; -import mask from "@src/core/mask"; -import * as task from "@src/core/task"; -import time from "@src/core/time"; -import hash from "@src/data/hash"; -import * as metric from "@src/data/metric"; -import {check} from "@src/data/token"; -import { queue } from "@src/data/upload"; -import * as boxmodel from "./boxmodel"; -import * as doc from "./document"; -import * as dom from "./dom"; -import * as target from "./target"; - -export default async function(type: Event): Promise { - let tokens: Token[] = [time(), type]; - let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; - switch (type) { - case Event.Document: - let d = doc.data; - tokens.push(d.width); - tokens.push(d.height); - metric.measure(Metric.DocumentWidth, d.width); - metric.measure(Metric.DocumentHeight, d.height); - queue(tokens); - break; - case Event.BoxModel: - let bm = boxmodel.updates(); - for (let value of bm) { - tokens.push(value.id); - tokens.push(value.box); - } - queue(tokens); - break; - case Event.Target: - let targets = target.updates(); - for (let value of targets) { - tokens.push(value.id); - tokens.push(value.hash); - tokens.push(value.box); - } - queue(tokens); - break; - case Event.Hash: - let selectors = dom.selectors(); - let reference = 0; - for (let value of selectors) { - if (task.longtask(timer)) { await task.idle(timer); } - let h = hash(value.selector); - let pointer = tokens.indexOf(h); - tokens.push(value.id - reference); - tokens.push(pointer >= 0 ? [pointer] : h); - reference = value.id; - } - queue(tokens); - break; - case Event.Discover: - case Event.Mutation: - let values = dom.updates(); - for (let value of values) { - if (task.longtask(timer)) { await task.idle(timer); } - let metadata = []; - let data: NodeInfo = value.data; - let active = value.metadata.active; - let keys = active ? ["tag", "path", "attributes", "value"] : ["tag"]; - for (let key of keys) { - if (data[key]) { - switch (key) { - case "tag": - metric.counter(Metric.Nodes); - tokens.push(value.id); - if (value.parent && active) { tokens.push(value.parent); } - if (value.next && active) { tokens.push(value.next); } - metadata.push(value.position ? `${data[key]}~${value.position}` : data[key]); - break; - case "path": - metadata.push(`${value.data.path}>`); - break; - case "attributes": - for (let attr in data[key]) { - if (data[key][attr] !== undefined) { - metadata.push(attribute(value.metadata.masked, attr, data[key][attr])); - } - } - break; - case "value": - let parent = dom.getNode(value.parent); - let parentTag = dom.get(parent) ? dom.get(parent).data.tag : null; - let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; - metadata.push(text(value.metadata.masked, tag, data[key])); - break; - } - } - } - - // Add metadata - metadata = meta(metadata); - for (let token of metadata) { - let index: number = typeof token === "string" ? tokens.indexOf(token) : -1; - tokens.push(index >= 0 && token.length > index.toString().length ? [index] : token); - } - } - queue(tokens); - break; - } -} - -function meta(metadata: string[]): string[] | string[][] { - let value = JSON.stringify(metadata); - let hashed = hash(value); - return check(hashed) && hashed.length < value.length ? [[hashed]] : metadata; -} - -function attribute(masked: boolean, key: string, value: string): string { - switch (key) { - case "src": - case "srcset": - case "title": - case "alt": - return `${key}=${masked ? "" : value}`; - case "value": - case "placeholder": - return `${key}=${masked ? mask(value) : value}`; - default: - return `${key}=${value}`; - } -} - -function text(masked: boolean, tag: string, value: string): string { - switch (tag) { - case "STYLE": - case "TITLE": - return value; - default: - return masked ? mask(value) : value; - } -} +import {Event, Metric, Token} from "@clarity-types/data"; +import {NodeInfo} from "@clarity-types/layout"; +import mask from "@src/core/mask"; +import * as task from "@src/core/task"; +import time from "@src/core/time"; +import hash from "@src/data/hash"; +import * as metric from "@src/data/metric"; +import {check} from "@src/data/token"; +import { queue } from "@src/data/upload"; +import * as boxmodel from "./boxmodel"; +import * as doc from "./document"; +import * as dom from "./dom"; +import * as target from "./target"; + +export default async function(type: Event): Promise { + let tokens: Token[] = [time(), type]; + let timer = type === Event.Discover ? Metric.DiscoverTime : Metric.MutationTime; + let addEventToQueue = true; + switch (type) { + case Event.Document: + let d = doc.data; + tokens.push(d.width); + tokens.push(d.height); + metric.measure(Metric.DocumentWidth, d.width); + metric.measure(Metric.DocumentHeight, d.height); + queue(tokens); + break; + case Event.BoxModel: + let bm = boxmodel.updates(); + for (let value of bm) { + tokens.push(value.id); + tokens.push(value.box); + } + queue(tokens); + break; + case Event.Target: + let targets = target.updates(); + for (let value of targets) { + tokens.push(value.id); + tokens.push(value.hash); + tokens.push(value.box); + } + queue(tokens); + break; + case Event.Hash: + let selectors = dom.selectors(); + let reference = 0; + for (let value of selectors) { + if (task.longtask(timer)) { await task.idle(timer); } + let h = hash(value.selector); + let pointer = tokens.indexOf(h); + tokens.push(value.id - reference); + tokens.push(pointer >= 0 ? [pointer] : h); + reference = value.id; + } + queue(tokens); + break; + case Event.Discover: + case Event.Mutation: + let values = dom.updates(); + for (let value of values) { + if (task.longtask(timer)) { await task.idle(timer); } + let metadata = []; + let data: NodeInfo = value.data; + let active = value.metadata.active; + let keys = active ? ["tag", "path", "attributes", "value"] : ["tag"]; + for (let key of keys) { + if (data[key]) { + switch (key) { + case "tag": + metric.counter(Metric.Nodes); + tokens.push(value.id); + if (value.parent && active) { tokens.push(value.parent); } + if (value.next && active) { tokens.push(value.next); } + metadata.push(value.position ? `${data[key]}~${value.position}` : data[key]); + break; + case "path": + metadata.push(`${value.data.path}>`); + break; + case "attributes": + for (let attr in data[key]) { + if (data[key][attr] !== undefined) { + metadata.push(attribute(value.metadata.masked, attr, data[key][attr])); + if (attr === "clarity") { + switch (data[key][attr]) { + case "no-track": + addEventToQueue = false; + break; + } + } + } + } + break; + case "value": + let parent = dom.getNode(value.parent); + let parentTag = dom.get(parent) ? dom.get(parent).data.tag : null; + let tag = value.data.tag === "STYLE" ? value.data.tag : parentTag; + metadata.push(text(value.metadata.masked, tag, data[key])); + break; + } + } + } + + // Add metadata + metadata = meta(metadata); + for (let token of metadata) { + let index: number = typeof token === "string" ? tokens.indexOf(token) : -1; + tokens.push(index >= 0 && token.length > index.toString().length ? [index] : token); + } + } + if (addEventToQueue) { + queue(tokens); + } + break; + } +} + +function meta(metadata: string[]): string[] | string[][] { + let value = JSON.stringify(metadata); + let hashed = hash(value); + return check(hashed) && hashed.length < value.length ? [[hashed]] : metadata; +} + +function attribute(masked: boolean, key: string, value: string): string { + switch (key) { + case "src": + case "srcset": + case "title": + case "alt": + return `${key}=${masked ? "" : value}`; + case "value": + case "placeholder": + return `${key}=${masked ? mask(value) : value}`; + default: + return `${key}=${value}`; + } +} + +function text(masked: boolean, tag: string, value: string): string { + switch (tag) { + case "STYLE": + case "TITLE": + return value; + default: + return masked ? mask(value) : value; + } +}